{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "instrumental-afternoon",
   "metadata": {},
   "source": [
    "# lesson-02\n",
    "* **从简单线性回归到复杂的神经网络**\n",
    "\n",
    "* **从手动书写梯度到自动梯度**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sufficient-neighbor",
   "metadata": {},
   "source": [
    "1、除常见的线性函数关系还要一种常见的函数关系是“S”型的一种函数\n",
    "$$\n",
    "sigmoid(x)=\\sigma(x)=\\frac{1}{1+e^{-x}}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "rocky-chase",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "noted-catholic",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "mental-butler",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_x = np.linspace(-10, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "proof-rachel",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "structured-vehicle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x28a198cc490>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfeElEQVR4nO3de3SU933n8fd3RjcQAmGQxB1hGzCXGBsTcEzq+BqD7YTam+w6bprETtfrXbtNzrZn48ZpmlO33TZp9rRJnRDHoU5ap068wTHrYi6Okzi2YwfMXQKMuAvQBQxIIKTRzHz3jxmwLI/QCEZ6Zkaf1zlz5rn8RvPlmZkPz/zmeZ6fuTsiIpL7QkEXICIimaFAFxHJEwp0EZE8oUAXEckTCnQRkTxRENQTjx492qurq4N6ehGRnPTWW28ddfeKVOsCC/Tq6mrWr18f1NOLiOQkM9vf0zp1uYiI5AkFuohInlCgi4jkCQW6iEieUKCLiOSJXgPdzJaZWZOZbethvZnZt8yszsy2mNnczJcpIiK9SWcP/Slg0XnWLwamJm8PAN+9+LJERKSvej0O3d1fMbPq8zRZAvzIE9fhfcPMys1srLsfyVSRIpKfYnGnIxojEo0TicbpSN46Y3GiMaczHicW93PzsXjiFo07cU9Mv3sPcXfc352OO4n5uONwbh7OtgWH5H1iviv3d5f5uWVn5/098929b3GXhvOqL+H6aSnPDboomTixaDxwsMt8fXLZ+wLdzB4gsRfPpEmTMvDUIhIUd6elPUpzawdNre0cOxXh5JlOWto7E/dnorQk59siMdoiMc5Eosn7GGc6Y0Tjg2s8BrPE/YMfuSxrA91SLEv5Krn7E8ATAPPmzRtcr6RIjnF3jp2OsP/YafYebWPf0dPsPXaaIyfO0NTaQXNrBx3ReMrHFoaNEUMKGT6kkLKSQkqLwowcWsiQogKGFoYZUhRmaFGY4oIwxYUhisIhigpCFBck7ovCIQrCIQrCRmEoRDhkFIaNcMgoCIUIhSAcMsKWWBZK3ptByCx5AwzCZlhy3jAwzrUzEtOGnQvbc/fJaEusP7suuYxubS1VDA68TAR6PTCxy/wE4HAG/q6IDBB3Z9+xNjYfPMGm5G130ylaO6Ln2oQMJowcyoSRQ5g3eSSVw0uoGFZM5fBiKoYVM7qsOBHiJYWUFIayJuQGk0wE+grgYTN7BlgAnFT/uUh2c3e2HWrh5R1NbDhwnM31JzjR1gnA0KIws8eP4K6546keVcqU0aVMHjWUCSOHUlSgI52zWa+Bbmb/DtwAjDazeuAvgUIAd18KrARuB+qANuC+/ipWRC5cNBZn3b7jrK5pYG1tI4dOnCFkMK2qjEWzxnDVxHLmTCxnauUwCsIK7lyUzlEun+plvQMPZawiEcmoTQdP8PQb+3lpeyPH2zopKghx/dTRfOGWqdwyo4pLSouCLlEyJLDL54pI/3F3fvV2M9/79W7e2PMOw4oLuHlGJbfNGsNHplVQWqyPfj7SqyqSRzpjcV7Ycpjv/XoPOxpaGTuihK/cMYN75k9imEI87+kVFskD7s5zGw/xzTVvc+jEGaZWDuMfPjmHj88Zpx8yBxEFukiOa2pp58vPbeWl7U3MmVjOXy2ZxY3TKwmFdNjgYKNAF8lR7s7zmw7zlytqaO+M8Rd3zuRz11UTVpAPWgp0kRzU3NrBo89tZU1tI9dMHsk3PnEll1YMC7osCZgCXSTHrNrWwJ8v38LpSIxHb5/B/R+eor1yARToIjnl3393gC8/t5UrJ5TzzU/O4fJK7ZXLuxToIjniB6/u5bEXarlxegXf/fQ1lBSGgy5JsowCXSTLuTuP/7KOf1jzNotnj+Gf7rlahyJKSgp0kSzm7nx99U6++6vd3H31eL7+iSt1nRXpkQJdJEvF485fvVDLU6/v4w8WTOKxJbN1bLmclwJdJAu5O19+bivPrDvIH314Co/eMUPXF5deKdBFstBTr+/jmXUHeejGy/izj05XmEta1BknkmU2HTzB367czi0zKhXm0icKdJEscrKtk4ee3kBlWQn/8Mk5CnPpE3W5iGQJd+fP/u9mmlrb+el/+xDlQzXwhPSN9tBFssQPXt3L2tpGHlk8g6snjQy6HMlBCnSRLLDhwHH+7sUd3DarivsXVgddjuQoBbpIwI6fjvDw0xsYW17C1z+hfnO5cOpDFwlQPO786bObOXoqws/++3WMGFIYdEmSw7SHLhKgn286xMs7mnj0jhl8YMKIoMuRHKdAFwnImUiMr6/ayZwJI/jDaycHXY7kAQW6SEC+/5s9NLS085U7Z+oaLZIRCnSRADS2tPPdX+3m9g+M4YPVlwRdjuQJBbpIAL65ZiexuPOlRVcEXYrkEQW6yACrOXySZ9+q57PXTWbyqNKgy5E8okAXGUDuzt/8x3bKhxTy8E1Tgy5H8owCXWQAvbyjidd3H+OLt0zTMeeScQp0kQHSGYvzNyu3c2lFKfcumBR0OZKHFOgiA+THbx5gT/NpHr19BoUaF1T6QVrvKjNbZGY7zazOzB5JsX6Emf0/M9tsZjVmdl/mSxXJXSfbOvnHl95m4eWjuOmKyqDLkTzVa6CbWRh4HFgMzAQ+ZWYzuzV7CKh19znADcA3zUwXcxZJ+sFrezne1smjt8/Uxbek36Szhz4fqHP3Pe4eAZ4BlnRr40CZJd6pw4B3gGhGKxXJUe2dMZ5+Yz83X1HJzHHDgy5H8lg6gT4eONhlvj65rKt/BmYAh4GtwBfcPd79D5nZA2a23szWNzc3X2DJIrllxebDHDsd4fMfnhJ0KZLn0gn0VN8Pvdv8bcAmYBxwFfDPZva+XRF3f8Ld57n7vIqKij6WKpJ73J1lr+7lijFlfOiyUUGXI3kunUCvByZ2mZ9AYk+8q/uA5Z5QB+wFdE6zDHq/3X2MHQ2t3L9wivrOpd+lE+jrgKlmNiX5Q+c9wIpubQ4ANwOYWRUwHdiTyUJFctGy1/YyqrSIj181LuhSZBDodcQid4+a2cPAaiAMLHP3GjN7MLl+KfAY8JSZbSXRRfMldz/aj3WLZL29R0/zix1N/PFNUykpDAddjgwCaQ1B5+4rgZXdli3tMn0Y+GhmSxPJbU+9tpeCkPHpa3VWqAwMna4m0g9Onunk2bfq+diccVSWlQRdjgwSCnSRfvDTdQdpi8S4f6EOVZSBo0AXybBoLM5Tr+9jwZRLmD1eAz/LwFGgi2TYmtpGDp04w/06kUgGmAJdJMOWvbqXSZcM5ZYZVUGXIoOMAl0kgzYfPMH6/cf53HXVhEM6kUgGlgJdJIN+9Nv9DCsu4JPzJgRdigxCCnSRDGmLRHlx2xE+NmcsZSUaXk4GngJdJEPW1DTSFonx+1d1vxipyMBQoItkyPKNhxhfPoQPVl8SdCkySCnQRTKgqaWdV3c1c9fV4wnpx1AJiAJdJANWbD5M3OGuuepukeAo0EUyYPmGQ8yZMILLKoYFXYoMYgp0kYu0s6GV2iMt3HW19s4lWAp0kYu0fGM94ZBx5xwNYiHBUqCLXIRY3Hl+42E+Mq2C0cOKgy5HBjkFushFeGPPMRpa2tXdIllBgS5yEZZvOERZcQG3ztSFuCR4CnSRC3QmEmPVtiMs/sAYjRkqWUGBLnKB1tQ2cDoS466rdSEuyQ4KdJEL9NzGQ4wbUcKCKTrVX7KDAl3kAjS3dvCbXUdZolP9JYso0EUuwIrNh4nFnbt1dItkEQW6yAX4+cZDzB4/nKlVZUGXInKOAl2kjw6+08bWQye580qdGSrZRYEu0keraxoAWDRrTMCViLyXAl2kj1Zta+CKMWVUjy4NuhSR91Cgi/RBU0s7bx04zuLZY4MuReR9FOgifbCmthF3WDRb3S2SfRToIn2walsDl44uZVqVBrKQ7JNWoJvZIjPbaWZ1ZvZID21uMLNNZlZjZr/ObJkiwTvRFuG3e45x2+wxmOlkIsk+Bb01MLMw8DhwK1APrDOzFe5e26VNOfAdYJG7HzCzyn6qVyQwa2sbicVdR7dI1kpnD30+UOfue9w9AjwDLOnW5l5gubsfAHD3psyWKRK81TUNjBtRwpUTRgRdikhK6QT6eOBgl/n65LKupgEjzexXZvaWmX0m1R8yswfMbL2ZrW9ubr6wikUCcKojyiu7jqq7RbJaOoGe6t3r3eYLgGuAO4DbgL8ws2nve5D7E+4+z93nVVRU9LlYkaD8ckcTkWhchytKVuu1D53EHvnELvMTgMMp2hx199PAaTN7BZgDvJ2RKkUCtqqmgdHDirhm8sigSxHpUTp76OuAqWY2xcyKgHuAFd3aPA/8npkVmNlQYAGwPbOligSjvTPGL3c08dFZYwjrUrmSxXrdQ3f3qJk9DKwGwsAyd68xsweT65e6+3YzWwVsAeLAk+6+rT8LFxkov9l1lLZITEe3SNZLp8sFd18JrOy2bGm3+W8A38hcaSLZYdW2BoaXFHDtpaOCLkXkvHSmqMh5dMbivLS9kVtmVlFUoI+LZDe9Q0XO4409xzh5plPdLZITFOgi57FqWwNDi8JcP02H2Ur2U6CL9CAed1bXNHLj9EpKCsNBlyPSKwW6SA82HjzO0VMdfHRWVdCliKRFgS7SgzW1jRSGjRuv0LXmJDco0EV6sLa2kWsvHcXwksKgSxFJiwJdJIW6plPsaT7NrTPV3SK5Q4EuksLa2kYAbpmhQJfcoUAXSWFtbQOzxw9nXPmQoEsRSZsCXaSb5tYONh48wa0zdDKR5BYFukg3v9jeiDvqP5eco0AX6WZtbSPjy4cwY2xZ0KWI9IkCXaSLtkiUV+uOcuvMKg01JzlHgS7SxStvH6UjGuej6m6RHKRAF+liTW0DI4YU8sEplwRdikifKdBFkqKxOC/vaOKmKyopDOujIblH71qRpPX7j3OirVNHt0jOUqCLJK2tbaQoHNK1zyVnKdBFAHdnbW0j110+imHFaQ21K5J1FOgiwNuNpzjwTpu6WySnKdBFSFy7BXQxLsltCnQREoNZzJlYTtXwkqBLEblgCnQZ9BpOtrOl/qROJpKcp0CXQW9NsrtFgS65ToEug96qbQ1cVlHK1CpdjEtymwJdBrV3Tkd4c+87LJqta59L7lOgy6D2Um0jsbizaNbYoEsRuWgKdBnUVtU0ML58CLPHDw+6FJGLpkCXQau1vZNXdx1l0ewxuva55AUFugxaL+9oIhKLs1j955In0gp0M1tkZjvNrM7MHjlPuw+aWczMPpG5EkX6x+qaBirKipk7aWTQpYhkRK+BbmZh4HFgMTAT+JSZzeyh3d8DqzNdpEimnYnE+OWOZm6bVUUopO4WyQ/p7KHPB+rcfY+7R4BngCUp2v0x8DOgKYP1ifSLV3Y1c6YzpqNbJK+kE+jjgYNd5uuTy84xs/HAXcDS8/0hM3vAzNab2frm5ua+1iqSMau3JYaaW3CphpqT/JFOoKf6Purd5v8R+JK7x873h9z9CXef5+7zKio0iIAEIxKNs3Z7I7fOrNJQc5JX0rmSfz0wscv8BOBwtzbzgGeSh36NBm43s6i7/zwTRYpk0m/3HKO1PcqiWTq6RfJLOoG+DphqZlOAQ8A9wL1dG7j7lLPTZvYU8ILCXLLVqm0NlBaF+fDU0UGXIpJRvQa6u0fN7GESR6+EgWXuXmNmDybXn7ffXCSbxOLO2toGbryikpLCcNDliGRUWoMnuvtKYGW3ZSmD3N0/d/FlifSP9fve4eipiC7GJXlJvwjJoPLitgaKCkLcOL0y6FJEMk6BLoOGu7O6poHrp1ZQWpzWl1ORnKJAl0FjS/1Jjpxs17VbJG8p0GXQ+I+tRygIGTfPUHeL5CcFugwKsbjz/KZD3DC9kvKhRUGXI9IvFOgyKLy++yiNLR3cPXd8741FcpQCXQaF5zYcoqykgJuuUHeL5C8FuuS9tkiUVTUN3HnlWJ1MJHlNgS55b3VNA22RGL9/lbpbJL8p0CXvLd9wiPHlQ/hgtS6VK/lNgS55ramlndfqjnLX1eM1MpHkPQW65LUVmw8Td7hLR7fIIKBAl7y2fMMh5kwYwWUVw4IuRaTfKdAlb+1saKX2SAt3Xa29cxkcFOiSt5ZvrCccMu6cMy7oUkQGhAJd8lIs7jy/8TAfmVbB6GHFQZcjMiAU6JKX3thzjIaWdnW3yKCiQJe8tHzDIcqKC7h1ZlXQpYgMGAW65J0zkRirth1h8QfG6FR/GVQU6JJ31tQ2cDoS466rJwRdisiAUqBL3nl2fT3jRpSwYIpO9ZfBRYEueWVnQyuv1h3lD66drFP9ZdBRoEte+ZfX9lJcEOLe+ZOCLkVkwCnQJW8cO9XB8o2HuHvuBEaWapg5GXwU6JI3fvzmASLROPcvrA66FJFAKNAlL0SicX70xn6un1bB1KqyoMsRCYQCXfLCC1sO09zawec/PCXoUkQCo0CXnOfu/ODVvVxeOYzrp44OuhyRwCjQJeet23ecmsMt3LewGjMdqiiDlwJdct4PXt1D+dBC7taZoTLIpRXoZrbIzHaaWZ2ZPZJi/R+Y2Zbk7XUzm5P5UkXe78CxNtbUNnLv/EkMKdJ1W2Rw6zXQzSwMPA4sBmYCnzKzmd2a7QU+4u5XAo8BT2S6UJFUfvjbfYTN+MyHqoMuRSRw6eyhzwfq3H2Pu0eAZ4AlXRu4++vufjw5+wag777S71rbO/nJuoPcceVYxowoCbockcClE+jjgYNd5uuTy3ryeeDFVCvM7AEzW29m65ubm9OvUiSFZ9fXc6ojyn0LdaiiCKQX6KkOG/CUDc1uJBHoX0q13t2fcPd57j6voqIi/SpFuumIxlj22l6umTySqyaWB12OSFZIJ9DrgYld5icAh7s3MrMrgSeBJe5+LDPliaT2o9f3U3/8DF+4eWrQpYhkjXQCfR0w1cymmFkRcA+womsDM5sELAf+0N3fznyZIu9653SEb728ixunV3D9NH3TEzmroLcG7h41s4eB1UAYWObuNWb2YHL9UuCrwCjgO8kTO6LuPq//ypbB7J9eepu2SIwv3z4j6FJEskqvgQ7g7iuBld2WLe0y/UfAH2W2NJH3q2s6xb+9eYB750/SRbhEutGZopJT/vfK7QwtDPPFW9R3LtKdAl1yxmt1R/nFjiYeuulyRg0rDrockayjQJecEIs7f/0f25kwcgifu6466HJEspICXXLCz96qZ/uRFh5ZfAUlhbpmi0gqCnTJeqc7onxjzU7mTirnjg+MDbockaylQJes971f76a5tYOv3DlT1zsXOQ8FumS13c2neOI3e/j4nHHMnTQy6HJEspoCXbLWmUiMh57ewNCiAp1EJJKGtE4sEgnC11bUsLOxlafum6/L44qkQXvokpV+9lY9P1l/kIduuJyP6HotImlRoEvW2dXYyld+vo0FUy7RGaEifaBAl6zSFonyP57eQGlxmG9/6moKwnqLiqRLfeiSNdydr/x8G3XNp/i3zy+gcrj6zUX6Qrs/kjWeXV/P8g2H+JObprLw8tFBlyOScxTokhU2HjjOXzy/jYWXj+JPNAqRyAVRoEvg3txzjE8/+SZVw0v4x/9yNeGQzgYVuRAKdAnUr99u5rP/8jvGlg/h2Qc/REWZLosrcqH0o6gEZnVNA3/8441cXjmMf/38fF3jXOQiKdAlEM9vOsT//Olmrpwwgqc+N58RQwuDLkkk5ynQZcD9ZN0BHlm+lQVTLuHJz36QYcV6G4pkgj5JMmAi0TjffnkX3365jhumV7D009dosAqRDFKgy4CoPdzCnz67me1HWvhPcyfwt3fPprhAYS6SSQp06VedsTjf+eVuvv3yLkaWFvH9z8zj1plVQZclkpcU6NJvdjS08Kc/3UzN4RaWXDWOr31sFiNLi4IuSyRvKdAl446e6mDZq3v5/m/2MGJIIUs/fQ2LZo8JuiyRvKdAl4zZf+w03//NHp5dX08kFmfJnHF89WOzuER75SIDQoEuF21r/UmWvrKbF7ceoSAU4u654/mv11/KZRXDgi5NZFBRoMsFOXziDGtqGli5rYHf7X2HsuICHrj+Mu5bWE2VLnsrEggFuqStrqmV1TWNrK5pYEv9SQAurxzGI4uv4N4FkxheorM9RYKkQJeUOmNxdja0sungCTYdPMFb+4+z9+hpAOZMLOd/LZrObbPGqFtFJIso0Ae5eNxpau1g79HT7D92ml1Np9h88ARbD52kIxoHYFRpEVdNLOe+hdXcOrOKsSOGBFy1iKSSVqCb2SLgn4Aw8KS7/1239ZZcfzvQBnzO3TdkuFbpo3jcOd4WoflUB00tHTS3dtDU2kFTaztHTrSz79hp9h07TXtn/NxjigtCzB4/gk9fO5mrJpZz1cRyJowcQuIlFpFs1mugm1kYeBy4FagH1pnZCnev7dJsMTA1eVsAfDd5L0nuTizuRONOPDkdizudMScajxONJdZFY3E6Y05HNEYkGicSi9PRmbyPxmiLxDgTSdwnpqO0RWK0tkdpae/k5JnOxH1bJ60dUdzfX0tpUZiqESVMGVXKwstHUz1qKNWjS6keVcq48iEaYEIkR6Wzhz4fqHP3PQBm9gywBOga6EuAH7m7A2+YWbmZjXX3I5ku+NdvN/PYC+8+tXdJrBTZ9Z4VZ9d3f4yfW+/vTvu7bc+2Obvezy53iCfXx+Pvtjsb2Oem3VMG68UqDBtDCsMMLSqgrKSAEUMKqRpewrSqMoYn50eWFlFZVkJFWTGVZcVUlBVTqqsbiuSldD7Z44GDXebref/ed6o244H3BLqZPQA8ADBp0qS+1grAsOICpleVvXehpZx8b5Nkl4Gdm3/vY96z3s4uN8zOru8yb4n7kL23TShkhMwIGYn7kGFAOLk8HOpyS84Xho2CcIiCkFEYDlEQNgpCIYoLQxSHQxQVdLmFQwwtKmBIUZihRWEKwxpwSkTelU6gp8rI7vub6bTB3Z8AngCYN2/eBe2zXjN5JNdMHnkhDxURyWvp7OLVAxO7zE8ADl9AGxER6UfpBPo6YKqZTTGzIuAeYEW3NiuAz1jCtcDJ/ug/FxGRnvXa5eLuUTN7GFhN4rDFZe5eY2YPJtcvBVaSOGSxjsRhi/f1X8kiIpJKWoc7uPtKEqHdddnSLtMOPJTZ0kREpC90mISISJ5QoIuI5AkFuohInlCgi4jkCfP+OCc9nSc2awb2X+DDRwNHM1hOpmRrXZC9tamuvlFdfZOPdU1294pUKwIL9IthZuvdfV7QdXSXrXVB9tamuvpGdfXNYKtLXS4iInlCgS4ikidyNdCfCLqAHmRrXZC9tamuvlFdfTOo6srJPnQREXm/XN1DFxGRbhToIiJ5ImsD3cw+aWY1ZhY3s3nd1v25mdWZ2U4zu62Hx19iZmvNbFfyPuOjYpjZT8xsU/K2z8w29dBun5ltTbZbn+k6Ujzf18zsUJfabu+h3aLkNqwzs0cGoK5vmNkOM9tiZs+ZWXkP7QZke/X2709eDvpbyfVbzGxuf9XS5TknmtkvzWx78v3/hRRtbjCzk11e36/2d11dnvu8r01A22x6l22xycxazOyL3doMyDYzs2Vm1mRm27osSyuLMvJ5TIyNmX03YAYwHfgVMK/L8pnAZqAYmALsBsIpHv914JHk9CPA3/dzvd8EvtrDun3A6AHcdl8D/qyXNuHktrsUKEpu05n9XNdHgYLk9N/39JoMxPZK599P4pLQL5IYketa4M0BeO3GAnOT02XA2ynqugF4YaDeT315bYLYZile1wYSJ98M+DYDrgfmAtu6LOs1izL1eczaPXR33+7uO1OsWgI84+4d7r6XxDXY5/fQ7ofJ6R8Cv98vhZLYKwH+M/Dv/fUc/eDc4N/uHgHODv7db9x9jbtHk7NvkBjZKijp/PvPDX7u7m8A5WY2tj+Lcvcj7r4hOd0KbCcxPm+uGPBt1s3NwG53v9Cz0C+Ku78CvNNtcTpZlJHPY9YG+nn0NCB1d1WeHDUpeV/ZjzX9HtDo7rt6WO/AGjN7KzlQ9kB4OPmVd1kPX/HS3Y795X4Se3KpDMT2SuffH+g2MrNq4GrgzRSrP2Rmm83sRTObNVA10ftrE/T76h563rEKapulk0UZ2W5pDXDRX8zsJWBMilWPuvvzPT0sxbJ+O/YyzRo/xfn3zhe6+2EzqwTWmtmO5P/k/VIX8F3gMRLb5TES3UH3d/8TKR570dsxne1lZo8CUeDpHv5MxrdXqlJTLLugwc/7g5kNA34GfNHdW7qt3kCiS+FU8veRnwNTB6Iuen9tgtxmRcDHgT9PsTrIbZaOjGy3QAPd3W+5gIelOyB1o5mNdfcjya98Tf1Ro5kVAHcD15znbxxO3jeZ2XMkvl5dVEClu+3M7PvACylW9cvA3mlsr88CdwI3e7LzMMXfyPj2SiFrBz83s0ISYf60uy/vvr5rwLv7SjP7jpmNdvd+vwhVGq9NkAPGLwY2uHtj9xVBbjPSy6KMbLdc7HJZAdxjZsVmNoXE/7K/66HdZ5PTnwV62uO/WLcAO9y9PtVKMys1s7Kz0yR+GNyWqm2mdOuzvKuH50tn8O9M17UI+BLwcXdv66HNQG2vrBz8PPl7zA+A7e7+f3poMybZDjObT+JzfKw/60o+VzqvTZADxvf4TTmobZaUThZl5vPY37/6XuiNRBDVAx1AI7C6y7pHSfwivBNY3GX5kySPiAFGAb8AdiXvL+mnOp8CHuy2bBywMjl9KYlfrDcDNSS6Hvp72/0rsBXYknxTjO1eV3L+dhJHUeweoLrqSPQTbkrelga5vVL9+4EHz76eJL4GP55cv5UuR1v1Y00fJvFVe0uX7XR7t7oeTm6bzSR+XL6uv+s632sT9DZLPu9QEgE9osuyAd9mJP5DOQJ0JvPr8z1lUX98HnXqv4hInsjFLhcREUlBgS4ikicU6CIieUKBLiKSJxToIiJ5QoEuIpInFOgiInni/wNrNjaYg/x3yQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(sub_x, sigmoid(sub_x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "metropolitan-milton",
   "metadata": {},
   "source": [
    "## 对sigmoid函数进行平移拉伸"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "advisory-thesis",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "sized-swift",
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_linear(x):\n",
    "    k, b = random.random(), random.random()\n",
    "    return k * x + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "favorite-fusion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7gElEQVR4nO29eZwcVbn//z5dvc6e2TJJJpN9jwTCEECQBBADXBb1fq8CggsgouDV613Ae1X0cvWi/q6AhMUYERG+oj9FRVEBFa4IQhKykoTsy0y22ffprep8/6jqZSY9SSfpme6eed7JeZ3tqeqnqqs+c/rUqXOU1hpBEAQh/3Fl2wFBEAQhM4igC4IgjBJE0AVBEEYJIuiCIAijBBF0QRCEUYI7Wx9cWVmpp06dmq2PFwRByEveeuutFq11Vaq6rAn61KlTWbt2bbY+XhAEIS9RSu0fqk66XARBEEYJIuiCIAijBBF0QRCEUYIIuiAIwihBBF0QBGGUIIIuCIIwShBBFwRBGCVkbRy6IAhjE9PShKMWoahJxNRETIuoqYlYVjwdtTSmZadNS2Nqp8y001prTAtMrbEsjaU1lgbLqYulLQ1ojQYsy461U4eT1mgntvOQKLNtErZ2HYPyiSnIk2cjP2Zi8qTK+qnlXDQ75btBp4UIuiAIx8W0NJ39Edp6Q7T1RujoC9MTitIdjNIdjNhxKEpPMEp/xKQ/bNIfMekLmwSdfChqOiJuEbXG7hoMStnx7UtniKALgpA5tNa09YY51BHkcGc/R7qCHOoIcqSzn8OdQVp6QrT3RWjvC3O8dXC8bhclfjeFPjcBj0GB1yDgNRhX4CHgdeN3u/B7DHxuF163C5/bwOukvYbCY7hwGy48sbTLjg2Xwu1SuJzYcIJLJcfgUokypRJ5pUABSilcyo4Vdj2KAfV2DAoVF11SlKl4uRqUT94mKTPCiKALwignalrsbelld3MPu5t72dNsp/c099AVjA6w9RiKmlI/E0oCzK0pYVyhh/ICL+WFXsYV2nFZwEux302x302R343PbWTpyITBiKALwihCa01jez8bGzvY2NDBxoZONh/spD9ixm3Gl/iYXlnENWdOZFplEbXjAkwo9TOhNEBFoReXK3stTOH0EEEXhDznaFeQ/93ezCs7mnhzTxutvWHA7gpZMLGED58zmTNqS5lZXcS0ykKK/Z4seywMFyLogpBnRE2LdQc6eGV7E69sb2br4S7Abnkvm1PNmXVlnFlbxpyaYrxuGZk8lhBBF4Q8YVdTNz9b28iz6xpp6QljuBRnTxnHv10+h2Wzq5k3oTirD+SE7COCLgg5TE8oyvObDvHTNQ2sO9CB26W4ZG411545iQtnVVIakO4TIYEIuiDkIPtbe3n0ld38esMh+iMmM6oK+fcr5/KBs2qpKvZl2z0hRxFBF4QcoqGtjxV/3sXP1zXidinef+YkPnTOZBbXlUl3inBCTijoSqnHgauAJq31wiFslgEPAB6gRWu9NHMuCsLo51BHPw+/vIufrW1AobjpvCl8ZtkMqkv82XZNyCPSaaE/AawAnkxVqZQqAx4BLtdaH1BKVWfMO0EY5XT2RfjOS9v5yeoGNJoPnzOZOy6eyYTSQLZdE/KQEwq61vovSqmpxzG5AXhWa33AsW/KkG+CMKp5ccsR/uNXb9PWG+ZD9bXccfFMascVZNstIY/JRB/6bMCjlHoFKAYe1FoP1Zq/DbgNoK6uLgMfLQj5R2tPiK/+Ziu/2XiIeRNK+OHHz2HhpNJsuyWMAjIh6G7gbOBSIAD8TSn1htZ6x2BDrfVKYCVAfX392J1yTRiTaK15fvNh7vn1FrqCEb5w2WxuXzpDXv4RMkYmBL0R+0FoL9CrlPoLsAg4RtAFYazS2hPi33+5mRe2HGVRbSnf+j/nMaemONtuCaOMTAj6r4EVSik34AXOBe7PwH4FYVSw7XAXt/5oLc09Ib54xVxuuXAabkNa5ULmSWfY4k+AZUClUqoRuAd7eCJa68e01tuUUn8ANgEWsEpr/fbwuSwI+cMLW47wTz/dQLHfzc9vP58zasuy7ZIwiklnlMv1adh8G/h2RjwShFGA1ppHXtnNt1/YzqLJZay86WzGy5hyYZiRN0UFIcMEIyb/9vNNPLfxEO8/cyL3/f0Z+D2yCIQw/IigC0IGOdoV5JNPrmXzwU7+7fI5fHrpDHllXxgxRNAFIUMc7Ojn+pVv0NoT4vs31fPe+eOz7ZIwxhBBF4QMcMgR8/a+ME9/8jzOnFyWbZeEMYiMnRKE0+RQRz/XrXyD9t4wP77lXBFzIWuIoAvCaTBAzG8VMReyiwi6IJwihzv7uf77IuZC7iCCLginwOFOu2Xe1hPmyVuWiJgLOYE8FBWEk6S9N8xHvv8mbT1hfnTLEs6qG5dtl4RcJhqGYAf0tydC2RQYPz/jHyWCLggnQShq8qmn3qKxvZ+nbj2XxSLmYwfLglAn9LXZoX+ouN1O93fY6XDPsfu64HNw2X9m3EURdEFIE601X/zFZlbvbePB685kybTybLsknA7hPuhrgb5W6G2141g+Htqg1ynrbwdtpt6XckFgHATK7bhkEoxfmMgHypzYCaWTh+WQRNAFIU2++6ddPLv+IP982WyuPXNStt0RBhMN24Lc22yHHifua7FFuTdW12KXRfpS78flhoKKRKia46TLE2WBcjsfGGfHvlJwZf+RpAi6IKTBL9c3cv8fd/D3i2u585KZ2XZn7BAJQm+TLc49R+3Q2ww9TUnC3WTbBDtT78PwQWGlHQoqoXK2k65IlMXTFeAvhTydrkEEXRBOwJt7Wrnr55s5b3o5//3Bd8ncLKeL1nb3RfcR6DliC3K3E8fyMfEeSqT9pVBYDUXVMH4BFF1s5wsrobDKCU7aV5y3An2yiKALwnHY29LLp556i9ryAN+7sV6WizsecaE+7IQjSfERW6BjsRk+dntPIRSPh6LxUD0Ppi+zBbuw2i4rcgS8sArcvhE/vHxABF0QhqCzP8LNT6zBUIonPr6E0gJPtl3KHpFgQqi7Dtmh+wh0H4Kuw3bcfSS1UPvLoHiCLdZTLoDiGjsUjU/ERePBVzTihzXaSGfFoseBq4AmrfXC49idA7wBfFhr/fPMuSgII49laf75ZxtoaOvjmdvOo66iINsuDR/hPkekG534IHQedETbEe++1mO38xTYQl0yESafZ4tzyURHsCc4Yl0DHlnYY6RIp4X+BLACeHIoA6WUAXwTeCEzbglCdnn0f3fzx21NfPXq+dRPzePhiWbEFuTORjt0NTpiHRPtRrubZDCBciidBMUTYVK9LdQlExMCXjwhrx8ejlbSWYLuL0qpqScw+yzwC+CcTDglCNnkrztb+J8Xt3PNool87N1Ts+3O0Ghtv4HY2QgdDdAZC45odzbaXSTogdsFxkFJrS3Yk5fYcUltQrRLJoInkI0jEk6T0+5DV0pNAj4AXMIJBF0pdRtwG0BdXd3pfrQgZJxDHf384zPrmVFVlP0RLVrbw/I6GqBjP3QcsAW744Aj4I0Q7h64jeGD0lo7zLgkkS6dZL/MUjIRvIXZOR5h2MnEQ9EHgLu01uaJLn6t9UpgJUB9fb0+rrEgjDDhqMVnnl5HOGrx2E1nU+gb5jEDWtsvuXQcSAh2PHZCNDhwG38plNZB+TSYdhGUTbaFOhYXVkk3yBgmE1dsPfCMI+aVwJVKqajW+lcZ2LcgjBj/9fxWNjR08OhHFjOjKkMjLkLd0L4f2vfZYt2+Pyk+AJHegfaBcfbETVVzYdb7oKzODjHR9pdmxi9hVHLagq61nhZLK6WeAH4rYi7kG79af5An/7afT75nGle8a0L6G1qm/dCxfa8t2oPD4NEh3mIYNwXKp8OMix3BnmKXlU4Gf0nGjkkYe6QzbPEnwDKgUinVCNwDeAC01o8Nq3eCMALsONrNF5/dzJJp5dx1+dxjDSL9tji37bWFOxa377Nb2cljr11uu8963FSYd7Uj1lMTITBOukSEYSOdUS7Xp7szrfXHT8sbQRhh+sMmdzy9jmpvkMcuKce97VfQtidJvPc4I0WS8JVC+VT7lfO5V9lCXT7NjktqwZD39YTsIFeeMHbob7cFunWPI9p7aNr5Nj/pO0Cl6oKnk2yLxtvdItMvdsR6mp0vnyatbCFnEUEXRhfBLmjbDa1OaNuTyPe3DTDtC9RwsKecwgmXUHnGWY5gT7db2vIaupCHiKAL+Uek3+4Sad1lh2QB720aaFtSCxXTYf41UD4DKmZA+Qz2WZX83SNrmTexhGduOw8MmXRLyH9E0IXcxDLtl2hadiWEu3WXLdqdDQx4+7FovC3Ws5fbgl0x086XT0v5xmMoanLHI6/jcbv47vVn4RYxF0YJIuhCdulrg5ad0LrTFuyWnYmuEjOUsPOV2EJddx5U3OgIt93aPtmhfv/9u3fYcqiLVR+tZ2KZvOIujB5E0IXhx4w4XSQ7oWWH0+reaYt3cr+2y2O3qitmwqzL7Lhylh1n6A3IP7x9hCde38ctF07jvfPHn/b+BCGXEEEXMkestd2ywwlOy7t9H1jRhF1htS3U8652BHuWHZdNGdYhf43tffzbzzdyRm1p6vHmgpDniKALJ4dl2q+uDxbulh0D34o0vPaIkep5MP9aR7Rn290kgbIRdzsctbjz/65Ha1hx/WJZeUgYlYigC6kJ9zpCHRPu7Yn+7eS+7diiu3OvsuPKpNa2y8ie/4P49gvvsKGhg0c+snh0L1YhjGlE0Mcysdn+Wrbbot28I9Hq7mxI2CmXPTa7cg7MfK8t3FVz7L7tgtxf/OGPW4/y/Vf38tHzp3DlyczTIgh5hgj6WMCynG6SHdC8PSHazdvtBRJieArs1nXd+VD5MaiabYt3+fS8XZT3YEc///z/b2TBxBL+/cp52XZHEIYVEfTRRDRkd4m0bHda207cunPgvNoFlXYLe8EHnNb2bLv1XTIJXKOnbzliWtz5f9dhWpqHb1iM35M7XUCCMByIoOcjwS6nb3s7NL+TEO/2faAtx0jZ82dXzoHpSxPdJJWz86KbJBN8+4XtrD/QwYobzmJqpazSI4x+RNBzleT+7Vg3SUy8uw8l7Fwee+TI+IWw8O9tAa+abY8q8Y7dh39/2naUlX/Zw43n1XHVGROz7Y4gjAgi6NnGspxX3GP920ndJcmrsXsKbaGedlGii6Rqjv2w0vBkzf1c5JDTbz5/Qglf+rv52XZHEEYMEfSRIhq2X2cf0L+93X7dPdKXsCuosMV6/rX2MmSxrpKSSTJlaxqEoiZ3/N91RKIWD39E+s2FsUU6KxY9DlwFNGmtF6ao/whwl5PtAT6ttd6YUS/ziXBv0hDApO6Stj0D35YsqbVb2lMusAW7ao4t5IUV2fM9z9Fac8+vt7D+gL0u6DTpNxfGGOm00J8AVgBPDlG/F1iqtW5XSl0BrATOzYx7OUxvS6KLpGVnQrgHjN827CF/VXPsF29iDyUrZ8t828PA028e4Jk1Ddxx8YyTWxdUEEYJ6SxB9xel1NTj1L+elH0DqM2AX7mBZUHngSTB3p54+SZ5UilPQWImwPj47TnO+G1v9vwfQ6zd18bXfrOFZXOq+MJlc7LtjiBkhUz3od8C/H6oSqXUbcBtAHV1dRn+6NMg0u+M396RGA4YmxUw2p+wK6iwW9fzrk50kVTNtrtPRtH47XzjaFeQTz+9jkllAR687iwMlzxrEMYmGRN0pdTF2IJ+4VA2WuuV2F0y1NfX66HshgWt7cmjBk8o1bID2veTWDBBQVmdLdzTljpzkzhdJdK/nXOEoia3P/UWfaEoT996LqUBGfEjjF0yIuhKqTOAVcAVWuvWE9kPK2bUfsGmNcVsgMnDAN1+e6z2xLPgjOuSXnOfMabHb+cTyQ9BH7txMbPHF2fbJUHIKqct6EqpOuBZ4Cat9Y7TdylN+tsTswHGFkto2emMJokk7AqrbOGef23igWTlbCidLN0keU7sIeidF8/k8oXyEFQQ0hm2+BNgGVCplGoE7gE8AFrrx4CvABXAI8oeJx3VWtcPl8O88zw894/Q15Ioi690MwvmXJEk3DMhMG7YXBGyx8vvNHHPc1u4eE4V/3TZ7Gy7Iwg5QTqjXK4/Qf2twK0Z8+hElNbC3CsTCyaMwEo3Qm6x/kA7n3l6HfMmFPPQDYvlIaggOOSfCk5YBNc8lG0vhCyxu7mHm59YQ3WJjx9+fAlFvvy7hAVhuJBOZCFvONoV5KM/WI3hUjx58xKqivNzjnZBGC5E0IW8oLM/wsceX01HX5gnPrGEKRXyWr8gDEZ+rwo5TzBi8skn17K7uYcffnwJCyeVZtslQchJRNCFnCZqWvzTTzewem8b373+LC6cVZltlwQhZxFBF3KWYMTkH3+ynhe3HuUrV83nmkWyUIUgHA8RdCEn6Q1Fue3Ha3ltVytfu2YBH3v31Gy7JAg5jwi6kHN09IX5xBNr2NTYyf/8wyL+/uzRM4FnPhAyQ3SGOumN9NIX7aMv0kd/tJ++SB99UTsdNsOEzFA8DpkhIlaEiBkhakWJWANjU5t2sMxEWptYloWFhaWPDRqN1npAOv7PSdv/B5U5JOe1dmIGxk4mKTn0FFPHq0tpr4e2v3nhzXz+7M+f1P7SQQRdyCmauu2hiXuae3nkI4tZvqAm2y6NCnojvRztO0pTXxPNfc0D0h2hDjrDnXSGOukKdRE0g2nv163ceA0vPsOHx/DgcdnB7XLHY7fLjaEMvC4vhtvAUHZwKReGy45duHC57FgpZZcpFwpl551yIF6WHAOJskF5J5soT6qPESs7EeokVw0bar+Lqxef1H7SRQRdyBka2vq48Qdv0twd4oefOIcLZsoD0JPB0hYHuw+yt2svezv3sq9rnx137qM1eOycecWeYioLKhnnG0dtUS0LKhZQ6i2l1FdKibeEIm8RBe4CCjwFFLgLKPQUUuApwGf48Bk+vIYXt0skJJeQb0PICbYc6uSWJ9bSF47y1K3nsrhO5uA5Eb2RXja3bGZD0wY2NG9gU9MmuiPd8foyXxnTSqdxUe1F1JXUUVNYw/iC8VQFqqguqKbAI7OKjjZE0IWsorXmmTUN3PPcFsoLvPz0U+czb0JJtt3KSSxtsal5Ey/tf4nVR1azo30HlrZQKGaUzWD5tOUsrFjI9LLpTC2Zyji//FEca4igC1mjLxzlS798m2fXH+Q9syp54MNnUlEkr/MnY2mLjc0beXHfi7y0/yWO9h3F4/KwuHoxn3zXJzmr+izeVfUuSrzyR1AQQReyxK6mHj7z9FvsbOrh8++dxWcvmSWzJiZxuOcwT217ij/s/QNN/U14XB4umHQBn1v8OZZNXkaxVxbzEI5FBF0YcZ7beIgv/mITPo/Bkzcv4T2zqrLtUs6wp3MPj29+nOf3PA/ARbUX8b6p72Np7VKKvEVZ9k7IddJZ4OJx4CqgSWu9MEW9Ah4ErgT6gI9rrddl2lEh/2nuDvHfv9vGs+sPUj9lHCtuWExNqT/bbuUEW1q38IPNP+CP+/+Iz/Dx4bkf5mPzP8aEIlmJSUifdFroTwArgCeHqL8CmOWEc4FHnVgQAHs+lh+/sZ/vvLiDYNTks5fM5B8vnYXHkMk+93bu5Zurv8lrh16j2FPMre+6lRvn30i5vzzbrgl5SDorFv1FKTX1OCbXAk9q+7WoN5RSZUqpCVrrw5lyUshfVu9t4yu/fpt3jnRz0ewqvnr1fKZXSdeBaZk8te0pHlr/ED7Dx+cXf54PzfmQ9I0Lp0Um+tAnAQ1J+UanTAR9DNPUFeS+37/Ds+sPMqkswGM3ns3yBeNP+k270ci+zn18+bUvs6F5A8smL+Mr532FqgJ5jiCcPpkQ9FR3aMpJDJRStwG3AdTV1WXgo4VcY09zD99/dS+/WNcIGu68eCZ3XDyTgNfItmtZx7RMnt72NN9d/128hpdvXPgNrpp+lfyREzJGJgS9EZiclK8FDqUy1FqvBFYC1NfXn9xMN0JOs/5AO9/73z28sPUIHsPF3y+u5VMXTWdqpawsBPYwxLtfvZt1TetYWruUr5z/FaoLqrPtljDKyISgPwfcqZR6BvthaKf0n48NwlGLl7c38YNX97J6Xxslfjd3LJvJx949Vdb7TGJX+y4+9cdP0Rfp4+sXfp2rp18trXJhWEhn2OJPgGVApVKqEbgH8ABorR8Dfoc9ZHEX9rDFTwyXs0L2MS3NG3taeW7DIX7/9mG6glEmlQX48lXz+fA5kynyyasNyWxo2sAdf7oDr+HlicufYE75nGy7JIxi0hnlcv0J6jVwR8Y8EnKOqGmxoaGD3246zG83HaalJ0Sh12D5ghquXjSRC2dVyhDEFPz14F/5witfoDJQycrLVlJbLPO6C8OLNKeEY7AszdbDXfxtdyt/29PK6r1t9ISieN0uLp1bzTWLJnLx3Gr8HnnQORTP73meL/31S8wcN5NH3/solQGZClgYfkTQxzhaa5q7Q2w53MXWQ11sbOjgzb1tdPZHAJheWci1Z07k/BkVLJ1dRbHfk2WPc5+ntz3Nfavvo358Pd+95LsytlwYMUTQxwiWpWnqDrG/tZf9bX3sbu5h66Euth3uoqUnHLerKy9g+YLxnD+jgvOnV8qr+SfJqs2reHDdg1wy+RK+tfRb+Ax5OCyMHCLoowDL0rT3hWnqDtmhKxiPD3b0s7+1jwNtfYSiVnwbj6GYPb6Yi+dUM39iCfMnlDB3QgmlAWmBnyov7X+JB9c9yBXTruAbF35DVvMRRhy54rKEZWlCUYtgxCQYNekPmwQjVjzdE4rSF47SEzLpDUXpDUXpDkbp6o/Q0R+hoy9MR3+Ezj47b1rHDusv9ruZVBZgWmUhS2dXMaWigLqKQqaUFzBpXEAeZGaQba3b+PdX/51FVYu494J7RcyFrJB3V92Oo908v+lw4lVUnViLW8dWALcXA4/nieePrdMaZ2VxsJwYNJYFptZ2mZVcrzGtpKDBtCxMSxM1NVFLE7WsRNq0iJiasGkRjtohYlpEUwjw8VAKirxuSgs8lBV4KAt4mVgWoKzAQ2nAQ1WRj+oSP9XFPqqL/VSX+OSh5QjR0t/CZ//8Wcr8ZTxw8QPSzSJkjbwT9J1He3jwTzsHlKnEwt7OSt+xMrsiOa/ieWc9bgWGS+GKb6dwKdveUAqXU2e47G1dSuFOKosHpfC6XRQYLtwu28ZjuDCc2Ot24TVsm3je7SLgMfB7DCd24XPSRT43hT43hT47HfAY8jJKDhIyQ3zu5c/RFe7iR5f/SEazCFkl7wT9ynfVsO++v8u2G4KA1pqvvf41NjVv4jvLvsO8innZdmnE0c5PXh37pWwl0tpKqtMMSNuvr4CO2cf3M2ifKfKp62LpWLmTt5J+iTu/irXzEz1mo63EsQzOD/W5x3zOcf091nbCzFLq5ldk/PvIO0GXVmpmSO8mSX3TxG5CsG+SdLZJWZe07fFu3OP6MviGTOHbKd18liNQg/xMtn27ZQttRz18uvKreF+v45XXth+7P2c/cT/jYpfsV9JnJ9clHwNJxzbA/yH2M7gufjzOfmJdkXH/hqizk/ZxpDpfwsmjYPH7poigA3Qc7WP/ltb0b8rj/aVM+uudfMEOvoAH/CU/0Q15GoI42Hcg6ThOQXgHCUpy6yn1fJjCMTjdbLgSXXa4FJY26Tc9zHe9G293gF37muy6WNed00WHcrr3YnVJ+bgNCuU8nx7KNrYfl2FnkrdXTj9irLHjcsXyqfeXXDeUf/HjTt6PyzkhyvkMe5Ok7WyfEscx6Lic7WO28e2St3E53aexdGw/sXMQO1exsti5duLk4zn2+JLS8e/o2LqhztEAH2N5V1LX7uDvY7BPMGBfw0HeCXpzQzd//dnOExvGSHViU5z0VF+i6wQXSPINGbMfeLHYdYkLE5TLhXInX9CD6gffRDG7QYISf26QdDHH86l8HXThDvR30EWafNHGbj4UrqQbaYCfsc8efLEPeWyJ42FAfdI5OaH4JG7uoW/cYwVlsCAc/4ZOfeM19TXx/l+/n8nFk3ni8icIuAPpX4+CMIzknaBPX1TFLf/zntStDlL8lRzGv4bC2ENrzb1/u5ewGeZbF31LxDzP0JaFGY06IYIZjWDF8pFIvM6KRgbZRR275LSdt0zzmG2sqJnYt5mwt20jzHvPxZy1/KqMH1/eCbrhcWF4ZPy0kB2e3/s8rzS+wr/U/wtTSqZk252cxzJNopEw0XDYFsOwLaLRSAQzErZFNOLko0npWF00OqDOHJSOOmkrGo2nzUgEy3S2S6qzHEEdDgyPB8PtxuX2YBgGLrcbw3DbsRNchh17AwW4vd5h8SPvBF0QskVLfwv//eZ/s6hqETfOuzHb7qSF1toWvnA4IaxOHAmHMMORgeVO2s7bdbZ9LB1JsnXKwkl2cYG2y7W2TuxkGtiC6bFjRzxjebfbg8vtxuPzEygqxnDy7qRtEsLqiQtsYl+OvRMn9u+O18VFOWkfMdFWLlfO9ASIoAtCGmit+a83/otgNMh/XvCfGK5Te2nLssy4YEZDISLhENFQKFEWsdOReFlS3eB0ZKjyCNEksT4dXIYbt9eD4fHi9njttNuD2+vF8Hjx+gMYxSW4Y+LqlBsej20fE80B6YQQG0n7jG2TLLQxcc0Vwcx1RNAFgSShDYWIhEJEQsEBgrum4Q32vv0Gt9ZeSftfN/NmeK0junaIhELHCHTEqYuXO63eU0EpF26v1wm+pLSdLyi1f8bbopuwMWJ5j3fANolyj23r8WB4k2w9XgyvB9cp/uESskNagq6Uuhx4EDCAVVrr+wbVlwJPAXXOPv8/rfUPM+yrMEbRWmOZUSLBEJFw0I5jghsKJqUTYpwsyJFgMC66ceENBePiGwkF0xLai6ikb+N6XmE9YIusx+9zxNOHx+eLC6mvsJDCceW4vd4B5Qk7L+6kck9SfXIcs3UZ0koVTkw6S9AZwMPAZdgLQq9RSj2ntd6aZHYHsFVrfbVSqgrYrpR6Wmt9er/3hLxBa50QzaAtqPE4lg6HksQ4Vp4Q4AHbJuWj4dBJP8xSLhcenx+Pz4fH58ft8zlpH/6iIltA42V+Rzxjdv4BIvzolpWsbl3L/Zd9lxmVs+JCLF0BQq6RTgt9CbBLa70HwFkM+logWdA1UKzsq7sIaAOiGfZVyABmNHqs0AaDhEP9trgG+xNiGgra+STRDTvbRoNBwnHRtutP5tXBWOt2sIh6fH4CJSUJUY3XOcHvs8XYn1TmCHPyfjLVon1x34s8H3qVz130ORbNPu+09ycIw0k6gj4JaEjKNwLnDrJZATwHHAKKgQ/rFI+3lVK3AbcB1NXVnYq/Y4LYyISY4IZjIhtMEtVgihZwvK5/kFg7AhwMYpkn8XdWqYRg+v14fX7cfj9ef4DC0rKBIusPDBBlj99/bD5JhA2PJ+dbt13hLr7+5teZXzGfjy/4eLbdEYQTko6gp7rrBjfFlgMbgEuAGcBLSqlXtdZdAzbSeiWwEqC+vj7vXz7XWhONhBPiGuwnEgo5AhwaIKwDypyWcDg0SJiD/fEWsLbSH+7lMgxHMG1R9TrCWVBSiqe6ZoCgepPFdlDsjYmuPxDvcsh10R1Ovr/p+7QH23nsvY/J/OZCXpDOVdoITE7K12K3xJP5BHCftico2aWU2gvMBVZnxMvTxLLMgX21x23dDur/HSzGA+pCJzXO1vB44mLpdWKPP0BReXmihet0Q3j9gQGt2mT7AWV+H4ZbVhnKNA1dDTy97WmunXntmJxFUchP0hH0NcAspdQ04CBwHXDDIJsDwKXAq0qp8cAcYE8mHY3R2tjAztWvJ4lxUl9vvN93oHCf7Fjc2EiEAeLr9xEork7Rog0kdSsE4q3jeEvXHxNvPy5DhoDlC/evux+3y81nz/pstl0RhLQ5oaBrraNKqTuBF7CHLT6utd6ilLrdqX8MuBd4Qim1GbuL5i6tdctwONx68ACv/fTHKJdrYKvVad36i4oprqhM6t9NxN5juhpi2ydawG6fT8bejnHWHV3HS/tf4jNnfobqgupsuyMIaaP0SYxMyCT19fV67dq1J72dGY2itZYhY8KwYGmLjzz/EZr6m/jN+39Dgacg2y4JwgCUUm9pretT1eXdLFeGM0eDiLkwHPxu7+94u/VtPrf4cyLmQt6Rd4IuCMNFf7SfB956gHnl87hqeuanNhWE4UYEXRAcfrz1xxztO8q/nvOvuJTcGkL+IVetIGBPjbtq8yourbuUc2rOybY7gnBKiKALArBi/QoiVoR/Ovufsu2KIJwyIujCmGdH+w6e3fks18+9XlYhEvIaEXRhzPPw+ocp9BTyqTM+lW1XBOG0EEEXxjRbWrbw54Y/89EFH6XUV5ptdwThtBBBF8Y0D214iFJfKTfNuynbrgjCaSOCLoxZ1jet57WDr3Hzwpsp8hZl2x1BOG1E0IUxy4r1K6jwV3DdnOuy7YogZAQRdGFM8ubhN1l9ZDW3vutWecVfGDWIoAtjDq01D61/iOqCav5hzj9k2x1ByBiyDIsw5vjrwb+ysXkjXz7vy/gMX7bdEUYIrTWYpr0aWDRqx7G8aaJNCywTLMsus6x4mbYse83ceJ0G7djE0loPrNPaqUvKaw2WxjulDt+MGRk/RhF0YUwRa51PKprEB2Z+INvujEq01uhQCKu/H93XhxUMYvUH0aGkOBhEB0PocAgrFEKHI+hQCB0O22XhMDoSsfORiF0fsdNEouho1C6POuloBKKmnTZNiETQpmmnY2UnsazjcFPxyVup/ud/zvh+RdCFMcWfD/yZbW3buPeCe/EYsnRfMlprdDCI2dmJ2dGB2dWF1dNjx909mN12bPV0Y/X2Yvb2YvX2YvX22XFfX1zAT1U8lceD8noTweNJlMXSHg+uggLwuO2824MyDJTbbZcZbpTbjXIb4HLKDRfKSMTKcNl1TozhQiXHLpddp1wJG6WctMtOu5w0ClxOXrnstFKgVMI2OY/CXVWZ0e8uRlqCrpS6HHgQe8WiVVrr+1LYLAMeADxAi9Z6aca8FIQMYGmLFRtWMLVk6piYHldrjdXZSbSlxQ6trZht7ZjtbUTb2jBb2zDb2zE7OzA7OjE7O9Hh4y/XqAIBXEWFGIVFuAoLcRUW4qmpsdMFBbgCAVRBAFfATrsKAnaZ34/L70f5/Sifz077/Lh8XpTPZwePxxZF4ZQ5oaArpQzgYeAy7AWj1yilntNab02yKQMeAS7XWh9QSsm6XULO8cK+F9jVsYtvXfQt3K78/nFqhcNEjx4lcviwHR85QvTIUSJHj2A2txBtbiba0pJaoJXCKCvDKC/HPW4c3qnTMMpKMUpLcZXasVFahlFagquoGKOkGFdxMUZREcojv2pymXSu6iXALq31HgCl1DPAtcDWJJsbgGe11gcAtNZNmXZUEE6HqBXlkQ2PMLNsJsunLs+2OyfECoeJNDYSOXhwQAgfPEjk4CHMlmOX7HWVlOAZX427qorAlLNxV1XZobIKd2Ul7opyjIoKjNJSlCxYPipJR9AnAQ1J+Ubg3EE2swGPUuoVoBh4UGv95OAdKaVuA24DqKurOxV/BeGU+P3e37Ovax/3L7s/Zxav0JEI4YYGwnv2EN5/gPCBA4QP7Cey/wCRw4ft0RExPB48EyfgnTQJ/8XLcE+YgKdmAp6a8bhravCMH4+rsDBrxyLkBukIeqrFOwevLO0GzgYuBQLA35RSb2itdwzYSOuVwEqwF4k+eXcF4eSJWBEe3fgoc8vnckndJSP++WZPD+Fduwjt3k1ozx7Ce/fZIt7QAKYZtzPKyvBMqSNw9tmUTp6Mp24y3ro6PJMm4a6qkv5l4YSkI+iNwOSkfC1wKIVNi9a6F+hVSv0FWATsQBCyzG93/5aG7gYeuuShYW2dW6EQ4d27Ce7YQWjnTkK7dhHauZPoocNxG+Xx4J06Bd/s2RRfvhzftGl4p03DO2UKRqnM9iicHukI+hpgllJqGnAQuA67zzyZXwMrlFJuwIvdJXN/Jh0VhFMhYkZ4bONjLKxYyNLazAy80loTbWoi9M47BLfvsOMd2wnv3RdvcSuPB+/06RQsPhvfh2bimz0L34wZeGprpf9aGDZOKOha66hS6k7gBexhi49rrbcopW536h/TWm9TSv0B2ARY2EMb3x5OxwUhHX6565cc6j3El8//sj0W+CTRpkl4/36C27YR2raN4NZtBN95B7OtLW7jmTQJ35w5FF92Gf45c/DNno23rs4e/ywII4jSOjtd2fX19Xrt2rVZ+WxhbBAyQ/zds39HTWENP77ixycUdB2NEtq9m+CWrQS3OuGdd9B9fbaBx4Nv1kz88+bhnzcf/9w5+ObMwSguHoGjEQQbpdRbWuv6VHXShBBGLb/Y8QuO9h3lvy78r2PEXIfDhHbton/LFoJbthDcuo3Q9u3oUAgAVVCAf948yj74Qfzz5+OfPw/f9OkorzcbhyIIaSGCLoxKgtEgqzav4uzxZ3NO+Vn0vx0T7q0Et2yxxTsSAcBVVIR//nzG3XCDLd4L5uOdMkX6uoW8QwRdGFVY/f2Etm/nlT/9kA+uPsKyPi87/vUciIl3cTH+BQsYd9NN+BfMJ7BgAZ66OhkSKIwKRNCFvMXs6CD4zjv2g8pt2whu3Up4716wLKYC4wvdlJxZi3/Z+/AvmI9/wQJ7lMkpPBwVhHxABF3IeWIjTQYME9y+neiRI3Ebd00N/nnzKFm+nL8WHeK7Pb/h/g8/Sd34s7LouSCMLCLoQs6gtSZ65Ij9Us7OnYR2OPHu3fGHlbjd+KZNo+Ccc+xRJnPn4p83D3d5OQCdoU6+/ewVLKp7D2eKmAtjDBF0YcTR0ag9h8nevYR27ya8Zy+hPXZsdXfH7dxVVfhmzWLcddfhmzMH/9w5eGfOxHWckSarNq+iJ9zD5xd/fgSORBByCxF0YVjQpknk8BHC+/cROXCA8L79hPfvtyegamiIP6QEW7i9M2ZQevVV+GbNssPMmRhlZSf1mQd7DvL0tqe5ZsY1zCmfk+EjEoTcRwRdOCW0aRJtbiZy+DCRg4ecqV4bCTc22vnDhweItgoE8NbZ6ygWX3oJ3ukz8M2YjnfatIy9mPPQenuuljvPujMj+xOEfEMEXTgGs6eXaHMT0aZme6GE5maiTU1Ejx4hcviIvZhCU9OAmQIBjIoKPLWTCCxcQMny5Xgm1+KdMhXv1Cm4q6uHdXTJ1tatPL/neW59163UFNYM2+cIQi4jgj7K0ZaF1d2dWCeyo8NOt7cTbW3DbGu149ZWom1tRFtbE6+6J6F8Ptw14/HUTKBwyRLcE2rs+bgn1OCZOBHPpEn2Oo9ZQGvNd9Z+hzJfGTcvvDkrPghCLiCCnqPoaBSrvx+rrx/d34fV12fne3vthXt7e7F6ehP5nm6srm57Id+ubszubqyuLsyurqEX7DUMjPJxuMsrcFeUE5g8GaN8HJ5qe9Ubd1LsKi7O2fHbrx16jTePvMndS+6m2CvzqghjlzEn6FprME20ZdmxaYEZRZumkzfRUTNepiNROx2NoqMmOhqBaBQdidhlkQg6Eosj6HA4ESelrXAIHQqjQyF0OIQVCqGDIaxQEN0fTIpD6P7+Ey7Wm4wqKMAojq39WIK7stLumy4pxlVSYq8fWVZmrxUZSzv5fH9D0rRMvvPWd6gtquVDsz+UbXcEIavknaD3/O//cuQb3wDTAsuyBdpKSpumnR5UFhPwIVurw4XHg8ubWNk8nnZio6gYVVWFy+dHBfy4fH5cAb+9unpBob1yeqGzmnoggFHkrLYeiwsKxvScI7/Z8xt2tu/k20u/jceQBYyFsU3eCbpRVkbgXWeAS6FcBrhcdlq5QCkwXPFy5XIl6l2GXWe4EzZOrDxucBkotwGGYZe5DXC7UYbbrjcMlNuDchsojwfldtsroDtxPHi9KI8X5XXyOdpNMRoIRoM8tP4hFlYsZPmU3F/4WRCGm7wT9MCiRUxatCjbbgg5wFPbnqKpr4n73nOf/OEUBCCtDlSl1OVKqe1KqV1KqbuPY3eOUspUSv2fzLkoCMdypPcIqzavYmntUs6pOSfb7ghCTnBCQVdKGcDDwBXAfOB6pdT8Iey+ib1UnSAMG1prvvHmNzAtk7uW3JVtdwQhZ0inhb4E2KW13qO1DgPPANemsPss8AugKYP+CcIx/OnAn3i54WU+c+ZnmFw8OdvuCELOkI6gTwIakvKNTlkcpdQk4APAY8fbkVLqNqXUWqXU2ubm5pP1VRDoDnfzjTe/wdzyudw0/6ZsuyMIOUU6gp7qadPglaUfAO7SWpspbBMbab1Sa12vta6vqqpK00VBSPDgugdpDbZyz/n34Hbl3TN9QRhW0rkjGoHk37W1wKFBNvXAM85Ig0rgSqVUVGv9q0w4KQgA65vW89PtP+XGeTeysHJhtt0RhJwjHUFfA8xSSk0DDgLXATckG2itp8XSSqkngN+KmAuZJGyG+errX2VC4QQ+e9Zns+2OIOQkJxR0rXVUKXUn9ugVA3hca71FKXW7U3/cfnNByASPv/04ezr38PClD1Pgyc4kYIKQ66TVCam1/h3wu0FlKYVca/3x03dLEBLs7dzLyk0ruXzq5VxUe1G23RGEnCW/Z2YSRj2Wtvja376G3+2XMeeCcAJE0IWc5nubvsdbR9/iX+v/lcpAZbbdEYScRgRdyFlebXyVRzc8ytXTr+b9M9+fbXcEIecRQRdykobuBu5+9W5mj5vNl8//sky+JQhpIIIu5BzBaJAvvPIFNJr7l91PwB3ItkuCkBfIq3ZCTqG15t437uWdtnd4+NKHmVwic7UIQrpIC13IKX62/Wc8t/s5Pr3o0zJEURBOEhF0IWfY2LyR+9bcx4WTLuT2Rbdn2x1ByDtE0IWcoLmvmS+88gXGF4znvvfch0vJpSkIJ4v0oQtZ52jvUW598Va6w9386PIfUeorzbZLgpCXiKALWeVwz2FuefEW2oJtfO+y7zGvYl62XRKEvEUEXcgaB3sOcssLt9AZ6uR7l32PRVWy+LcgnA4i6EJWaOhq4JYXb6En0sOq961iQeWCbLskCHmPCLow4uzv2s/NL9xM2Azzg/f9QLpZBCFDyFACYUTZ2b6TT/zhE0TMCKvet0rEXBAySFqCrpS6XCm1XSm1Syl1d4r6jyilNjnhdaWUdIYKA9Ba87PtP+P6569Ho3l8+ePMKZ+TbbcEYVRxwi4XpZQBPAxchr2+6Bql1HNa661JZnuBpVrrdqXUFcBK4NzhcFjIPzqCHdzz+j38ueHPvHviu/n6hV+XqXAFYRhIpw99CbBLa70HQCn1DHAtEBd0rfXrSfZvYC8kLQisPryaL/71i7QF2/iX+n/hpvk3yUtDgjBMpCPok4CGpHwjx2993wL8/nScEvKfiBXh0Q2PsmrzKqaUTOGhKx9ifsX8bLslCKOadAQ91UTUOqWhUhdjC/qFQ9TfBtwGUFdXl6aLQj6hteblhpdZsWEFO9t38sFZH+Suc+6ShZ0FYQRIR9AbgeQ5TGuBQ4ONlFJnAKuAK7TWral2pLVeid2/Tn19fco/CkJ+orXm9UOv89D6h9jSuoUpJVN4YNkDXDrl0my7JghjhnQEfQ0wSyk1DTgIXAfckGyglKoDngVu0lrvyLiXQk6z5sgaVqxfwbqmdUwsnMh/vvs/uXrG1bhd8pqDIIwkJ7zjtNZRpdSdwAuAATyutd6ilLrdqX8M+ApQATziLBUW1VrXD5/bQrbpCffwpwN/4le7fsXao2upDlTzpXO/xAdnfRCP4cm2e4IwJlFaZ6fno76+Xq9duzYrny2cGhErwusHX+f5Pc/zcsPLBM0gtUW1XD/3ej4050P43f5suygIox6l1FtDNZjlN7FwXDqCHbx19C3+dvhvvLjvRdpD7ZT5yrh25rVcPeNqzqg8QxZwFoQcQQRdGEBbsI23jr7F2iNrWXN0DTvbdwLgN/wsnbyUq6ZfxQUTL5BuFUHIQUTQxyh9kT52d+xmV8cuO+7cxa72XRztOwpAwB1gUdUilp+5nHNqzmFh5UK8hjfLXguCcDxE0EcZpmXSFe6iPdROZ6iTpr4mjvQeGRj6jtDS3xLfxuvyMr1sOvU19cwqm8XZ489mQcUCaYULQp4hgj6MaK2xtIWlLaI6immZmNokakXjccSKEDEjRHWUiBkhYkUIW2FC0RAh0w5BM0goasd9kT56I730RHri6d5oL10hW8S7Ql3oFO99BdwBagprqCmoYXb5bCYWTmRm2UxmjptJbVEthsvIwhkSBCGT5J2gv3bwNb615lvxfLJ4DTViJ2YTqx+8Tap6jY7XDY6BuFBrrbGwBoi3pS1MbaYU1tPFpVwUegrt4LbjAk8BEwonUOYrY5x/HGW+MjvtG0dFoIKawhpKvCXy8FIQRjl5J+iFnkJmls0cUJYsVIrU6VgyVjZ4G4U6tkwpXMo1YBtFosylXPEypRSGMnApVzwYykAphVu5MVwGhjJwu9wYysBwGbiVG4/hweNKBLfLjcflwe/24zN8+A0/XsMbz/sMnwizIAgpyTtBP7P6TM6sPjPbbgiCIOQcMo+pIAjCKEEEXRAEYZQggi4IgjBKEEEXBEEYJYigC4IgjBJE0AVBEEYJIuiCIAijBBF0QRCEUULWFrhQSjUD+09x80qg5YRWI0+u+gW565v4dXKIXyfHaPRrita6KlVF1gT9dFBKrc3FJe5y1S/IXd/Er5ND/Do5xppf0uUiCIIwShBBFwRBGCXkq6CvzLYDQ5CrfkHu+iZ+nRzi18kxpvzKyz50QRAE4VjytYUuCIIgDEIEXRAEYZSQs4KulPoHpdQWpZSllKofVPdFpdQupdR2pdTyIbYvV0q9pJTa6cTjhsHHnyqlNjhhn1JqwxB2+5RSmx27tZn2I8XnfVUpdTDJtyuHsLvcOYe7lFJ3j4Bf31ZKvaOU2qSU+qVSqmwIuxE5Xyc6fmXzXad+k1Jq8XD5kvSZk5VSLyultjnX/+dS2CxTSnUmfb9fGW6/kj77uN9Nls7ZnKRzsUEp1aWU+vwgmxE5Z0qpx5VSTUqpt5PK0tKijNyPWuucDMA8YA7wClCfVD4f2Aj4gGnAbsBIsf23gLud9N3AN4fZ3/8BvjJE3T6gcgTP3VeBfzmBjeGcu+mA1zmn84fZr/cBbif9zaG+k5E4X+kcP3Al8HvsBQzPA94cge9uArDYSRcDO1L4tQz47UhdTyfz3WTjnKX4Xo9gv3wz4ucMuAhYDLydVHZCLcrU/ZizLXSt9Tat9fYUVdcCz2itQ1rrvcAuYMkQdj9y0j8C3j8sjmK3SoAPAT8Zrs8YBpYAu7TWe7TWYeAZ7HM2bGitX9RaR53sG0DtcH7eCUjn+K8FntQ2bwBlSqkJw+mU1vqw1nqdk+4GtgGThvMzM8yIn7NBXArs1lqf6lvop4XW+i9A26DidLQoI/djzgr6cZgENCTlG0l9wY/XWh8G+yYBqofRp/cAR7XWO4eo18CLSqm3lFK3DaMfydzp/OR9fIifeOmex+HiZuyWXCpG4nylc/xZPUdKqanAWcCbKarPV0ptVEr9Xim1YKR84sTfTbavq+sYumGVrXOWjhZl5LxldZFopdQfgZoUVf+htf71UJulKBu2sZdp+ng9x2+dX6C1PqSUqgZeUkq94/wlHxa/gEeBe7HPy73Y3UE3D95Fim1P+zymc76UUv8BRIGnh9hNxs9XKldTlA0+/hG91gZ8sFJFwC+Az2utuwZVr8PuUuhxno/8Cpg1En5x4u8mm+fMC1wDfDFFdTbPWTpk5LxlVdC11u89hc0agclJ+VrgUAq7o0qpCVrrw85Pvqbh8FEp5QY+CJx9nH0ccuImpdQvsX9enZZApXvulFLfB36boird85hRv5RSHwOuAi7VTudhin1k/HylIJ3jH5ZzdCKUUh5sMX9aa/3s4Ppkgdda/04p9YhSqlJrPeyTUKXx3WTlnDlcAazTWh8dXJHNc0Z6WpSR85aPXS7PAdcppXxKqWnYf2VXD2H3MSf9MWCoFv/p8l7gHa11Y6pKpVShUqo4lsZ+MPh2KttMMajP8gNDfN4aYJZSaprTsrkO+5wNp1+XA3cB12it+4awGanzlc7xPwd81Bm5cR7QGfvpPFw4z2N+AGzTWn9nCJsaxw6l1BLs+7h1OP1yPiud72bEz1kSQ/5SztY5c0hHizJzPw73U99TDdhC1AiEgKPAC0l1/4H9RHg7cEVS+SqcETFABfAnYKcTlw+Tn08Atw8qmwj8zklPx35ivRHYgt31MNzn7sfAZmCTc1FMGOyXk78SexTF7hHyaxd2P+EGJzyWzfOV6viB22PfJ/bP4Ied+s0kjbYaRp8uxP6pvSnpPF05yK87nXOzEfvh8ruH26/jfTfZPmfO5xZgC3RpUtmInzPsPyiHgYijX7cMpUXDcT/Kq/+CIAijhHzschEEQRBSIIIuCIIwShBBFwRBGCWIoAuCIIwSRNAFQRBGCSLogiAIowQRdEEQhFHC/wO1zo+TkX1ZVQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for _ in range(6):\n",
    "    plt.plot(sub_x, random_linear(sigmoid(random_linear(sub_x))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "infrared-excellence",
   "metadata": {},
   "source": [
    "上面将标准的sigmoid函数进行了横轴纵轴两个方向的拉伸"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "behavioral-installation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABhcElEQVR4nO3dd3hUVcLH8e+dyUx6770RQq+hg4AgVUHsYsfu+qrr2sva69q7WHZVUERQAenSe09CSyAV0nubTKae948JSEkggSSTxPN5njxTbjtzk/xyc+4pihACSZIkqeNT2bsAkiRJUsuQgS5JktRJyECXJEnqJGSgS5IkdRIy0CVJkjoJB3sd2M/PT0RFRdnr8JIkSR3Snj17SoQQ/g0ts1ugR0VFsXv3bnsdXpIkqUNSFCW7sWWyykWSJKmTkIEuSZLUSchAlyRJ6iRkoEuSJHUSMtAlSZI6ifMGuqIo3yqKUqQoyoFGliuKonykKEqaoijJiqIMaPliSpIkSefTlCv0/wGTzrF8MhBX/3UP8PnFF0uSJElqrvO2QxdCbFQUJeocq0wHvhe2cXi3K4ripShKsBAiv6UKKUmS1JqEEGA2YzUYESYjwmhCmEynPzcawahHGPUIg8H2aKwDY/02ZiPUPwqzCUwm26PFjLBYwGxCmC1gNeM8ZARu029v8c/REh2LQoHjp7zOqX/vrEBXFOUebFfxREREtMChJUmSTmExQV0VGCqxlBRQ9Nl/sVRWYtXrEXV1tsA2GOuD24LVZEGYrQiTtU2L6VtV0m4DXWngvQZnzRBCzAZmAyQkJMiZNSRJapzZCLpi0BVBTTHUFEJtCejLobbM9njieV0F1FWCqfbk5hWHXalI8sTR04SiFqgcBGq1QKUWKB4qVBoHFK0DisYBReuESqtF0WhQHB1tj1qt7UvjhOKoRdE6omgcUbSOoNXWP3dC0ZzyWqMFje05GkcUB0cUrRYcNCgO2vr3tODk2SqnrCUCPQcIP+V1GJDXAvuVJKmzMtVBRTZU5kBVHlTl2r4qc22vawpsYd0QtRacfcDFx/boGwvO3raQdPIERw+E1oPyDZ/j3NOLqI9fBq0baF3/elSp2/bztpGWCPTFwIOKoswDhgCVsv5ckiTMBig5CiWpUJYJ5ZlQlmV7rMrjrH/k3QLBI8QW0FEjwDUA3Pxt75947uJnC2SloYqBv9Ru3YopvwT/R5+AkP6t9hEbYjJYqCmvo7qsDl2FEV2lAV1FHRXlNVSX16GvMhE5xJMJV7V8g8DzBrqiKD8BYwA/RVFygBcADYAQ4gtgGTAFSANqgTtavJSSJLVfViuUZUBBEhSlQPFh22NZBgjLX+u5BYJ3NERfYnv0jgKvcPAIBfdgcNC2WJHKf56P2ssL9wkTWmyfJxjrzFSX1lFZrKe0sIqiwnIqSmupLTdirBJQd/bVf51aR622ilptFTptJTqrPxOwQ6ALIW48z3IB/KPFSiRJUvslhK2aJG8v5O6FvH2QlwiGSttyRWUL64Du0GO67dE/HnxibFfWbcBcXEz1mjX43HILKkfHC9qHxWyltLCarOw88nNLKSvQUVtixlqpRlV3+h8eg1pPjWM5NdpyajzL0QdWoXK3oPVU4erliIeXC77u3oQ4+eDrFIuvsy/h7uGNHPni2G34XEmSOgAhbFfaWZsga7Ptq7q+RlWlgcCe0OsqCB0AwX3BLx40TnYtcsXCX8Fsxuu6a8+5nhCCktpS0rKzOZZVSEluDfpCK5Q74ljrjkr81U1Hp9FR4VyMwbsaPEw4eatw93PEN9CDcG9/Aly74O/sj5+zHx5aD5TzVAm1FhnokiSdrqYIjq6CjA31AV7fxsEtEKJGQsQwW4AH9LR7eJ9JWCxUzJ+Py9ChOEZHA6Az6ciqzCKjPJPszALKjtViLFSjKXfHszYAjVULOGFFi9mlDJNnBaaoMtwDtPiFeBAeHkC4TxeCXIPQqi+sWkhYrVSXlVCel0d5fi7eYZFE9uzVgp/cRga6JP3dCQElRyBlKaQuh5xdgLDdiIwaCdGjIGoU+HY5781Ie6ox1pCxYgGavDy2zOjKzl//Se1xgVOZN/66cPx0YahFDL6AWWsA3zpcuhjxD3MkMjqIrjGRuDq7XFQZDLW1lOUepzT3OPlZ2eQdO0ZlYT6m8iKwmP9asdcl/EsGuiRJLUIIKNgPyT9D6jJbtQpAcD8Y8zR0mwKBvdplgFuFldzqXFLKU0gpSyG1LJUjpUcwFau5aVM0zn3uovhIF3ocdLdtoLHiHKwQOMCTmC7BhET74OHndFHVIsY6PSXHsinOziQ7PYPCY9lUF+QhdBUn17GgolLjSYXGk0q3nuDhj3tgMP6hYYztH3uRZ6FhMtAl6e+kugD2/wKJP0HRQVs9eMxoGPYgdJ0EnqH2LuFphBAcqz7G/pL9HCw5yKHSQ6SWp6Iz6vDWBxFR2Z043QDiy6ehMjtg8AKVQx19+0cS3MWL4C6e+AS5oqguLLyFEFQVF1KYmc6xI0c5np5BRW421qrSk+sYFQ3lGm/KtAHoA3rgHBBCQFgEkdFh9A/wJNrPhXAfFxwdWr/tuwx0SerszAZI+cMW4ulrQFghNAGmvgs9r7J10GknyurKSCxK5EDJAVuIlx6k2lgNgDueDDaM4/rKyTgV+GHV2W5aegY4Ez7cB7eMHah++YKeS+ejDQtr9rGF1Up5QR4F6UdJP5xKTtpRdPnHUIx6AKwoVGi8KNH6ovPviktQGMHR0cTGRDA20J1Yfzf83LR2uyEKMtAlqfOqLYPd38DOr2zd5j3CYOQ/oc8N4N/V3qVDCEFWVRaJRYnsK9rHvqJ9ZFVlAeCgOBDnHcekqEn08utFL79eZP5qJG1vEY6uDoR38yG8uw9h3b3x8HVGmM2kjbsXx8G9mhzmtZUV5B9NJfXAQbJTUtDlZp4Mb7OipkTrS4lTDEpoKH5RsXTp2oUhYT7EB7kT4O5o1+BujAx0SepsStJg+2eQ+COY9RA7DqZ/BrGXgsp+c9oIIcisymRX/i52Fuxkd+FuyurKAPB09KS/f3+u7HIl/QP608O3B04Op7eg2XVsO5G9fZlyfx9UZ1Sh1GzYgLmwkKDnn2v42FYrZXk5pO/fz6F9SZRmpEC17dhWFEq1PhQ7RaOOjCQoNo74+DgmhXsTH+SOk6bjDBMgA12SOouc3bDxHTiyAtQa6HMdDP0HBPawW5EKdAVszdvKjvwd7CrYRbG+GIBAl0BGhIxgYOBA+gf0J8ozCpXS+B8bk8FCRVEtcYMCzwpzgPKff8YhIAC3MWMAsFotFGaksX/XHtIPHKDm2FFU9VfftSpn8p2CMEf0xje6K917dWNiTCDxQe5tUs/dmmSgS1JHV5wKa1621ZO7+MLoJ2DQXeAW0OZFMVqM7Cncw5bcLWzO3Ux6ZToAvk6+DA4azODgwQwOGky4e3izqizK8nQgwC/M7bT3hRDUHTxEzabNqG+9iVW//srRxH3os4+iMtcBUK7xpNglGqeuXYjq0ZPRvePoH+GNu5Om5T54OyEDXZI6qsocWP+GrWpF4wpjn4Oh94Oj2/m3bUFFtUWsP76eDTkb2FWwC71Zj0alYWDgQGbEzWBEyAhivWIvqs65NLcGAA91DVUr91F34AAV+5PIzs6kWKNQ0iMSU+I2SNxGhYMHpR5d8IjpTvcB/bm8ZzRdAtxQX2BLl45EBrokdTS1ZbD5PdgxGxAw5H4Y9S9w9W2TwwshyKzMZO3xtaw7to7kkmQAwtzCuLLLlYwMHUlCYAIumovrpGOpqkKfvB99chJZSWpUllBSbr+aEndnijxcqHZ2hAAPzDhQ6hqOQ3x/ug8cwFX94ojydWmXNy1bmwx0SeoohICkebDyGdtY4X1vhLFPg1frz/4lhCClLIWVWStZc2zNydYovXx78VD/hxgbPvairsKF1YoxPZ3affvQ792HPikJY2YmJpWKIg8XMqITMFhXsKNLCFYUipyDUYV3p+vABC4Z2pcuge5/ywA/kwx0SeoIyjLhj39CxjoIGwyXvw9BLd91/Exp5WmsyFrByqyVZFVloVbUDA4azM3db2ZM+BgCXQMvaL9WgwF9UhL6vfuo3bcXfWIS1krbiI0GP1+yw8M4ltAPg6kGBQGiBL17PC5DEhg1ahgD44LRqO3XYqe9koEuSe2ZxQzbPoH1b4LKAaa8Awl3tmrzw/yafP7I+INlmctIq0hDpagYFDiIW3veyviI8Xg7eTd7n1adjtrERGp37aJ2927qkpIRJhMA2thYTCNHkqQ4kF1WhKOuCKw1VDh4Q5cRdO09iMp1gokzu9FrdPM7DP2dyECXpPYqbx8s/j/bmCvxU2HKf1qta77erGfNsTUsSlvEjvwdCAT9A/rz9OCnmRA1AT9nv2btz1pXh37vXnTbd6DbsZ26g4fAbAa1GqcePfC++WZKwqPYmFdMYWoy7jkptnI4B6DqN4nBl45hVEIPNGoVWftLWLo+Gd/Qtr3Z2xHJQJek9sZqtV2Vr3nJNuXadT9A9ytafKAsIQSJxYksSlvEiqwV6Ew6Qt1Cub/v/VwRewVh7k2/GhZmM/r9+6ndvh3d9h3o9+1DGI2gVuPcuze+s2bhMmgQJSERLPtzE/l7NuO1d6NtW9cAlIQpjJownoQ+cWfVhZ9o4eIjA/28ZKBLUntSWwa/32/rHNR9Gkz7GJy9WvQQ1cZqlqQv4Zcjv5BWkYazgzMTIicwvct0BgYOPGcHn1MZc3LRbd6MbssWdNu3Y622jbni2L073jNn4jpsKM4DEygxwaJla8mcvwjP4iM4CAtaZx+0g6cwbsokenSPOedxSnJqcPd1wtFZxtX5yDMkSe3F8Z3wyx2gK4LJ/4HBd7foVfmh0kPMT53Pssxl6M16evr25OXhLzMxamKTmhha6+qo3bGDmk2b0W3ejDErCwCH4GA8Jk3EdcQIXIYMwcHbG73Rwh8b97Hn5fdxO56Is7UOdwdnHHsO55LJk+g/sE+TW6WU5tSc1aFIapgMdEmyt1OrWDxCYdZK24xALcBkNbEyayVzD83lQOkBnNROTImZwnVdr6OnX8/zb5+XR82GDdSs34Buxw5EXR2KkxMugwfhfeMNuI4ciTYmBkVRsFoF24/ks/Kb39Ef2EqgPh9fRYUS2YvBEyYyfPQI1A7NixyzyUJFYS2xA9q+12tHJANdkuyprhJ+vafFq1gqDZUsOLKAH1N+pKi2iCiPKJ4a/BRXxF6Bh9aj0e2E1Urd/v1Ur1lLzfr1GI4cAUATHo7XNdfgNno0LoMHnTb5ckFlHT+v3MGRjasILTmMhzDi4u5H5OTrmTT9cty8m98q5oTy/FqEQN4QbSIZ6JJkL5U5MPda2/Rvk9+GwfdcdBXL8arjzDk8h9/SfkNv1jMkeAgvDHuBkaEjG60btxqN1O7YQfWfa6hZuxZzcTGo1bgkJBDwxBO4jRmNNjr6tCoSs8XKukP5LFv6J8rhzYTW5ROlcsCjx0DGzZhGbK+mV6mcS0mOrV5eVrk0jQx0SbKHgv22MDfq4OaFEDPmonaXWpbK7OTZrM5ejVqlZkr0FG7tcSvxPvENrm/V6ajZuJGqVavQbdyEVadD5eKC66hRuI8fh9sll6D29Dxru7wKPT9tPMT+NSuJLk4izKJDuHnTe8bNjLp8Ks5u7hf1Oc5UmqPDQaPCw9+5RffbWclAl6S2lrYG5t8GTh4wawUEnr8uuzH7i/czO3k263PW46pxZVavWczsPpMAl7PrnC01OmrWr6d65UpqNm1C1NWh9vXFY8oU3MePw2Xo0NOqUk4QQrAjs4wfl++gdu8autYcpTdWXGN6MHbGVcQlDEKlap1hZ0tya/AJdWtwyFzpbDLQJakt7ZsDSx4G/25w0y/gEXJBu9ldsJvZybPZlr8NT0dP/tHvH9zY7UY8HU+/qrbqdFSvXUfV8uXoNm9GGI04+PvjdfXVuE+cgMvAgSjqhsNYb7Tw274clqzYiF/mNqL0xxAOGrpcMp5RV87ANzT8gsreVEIISnNqiOnXvE5Nf2cy0CWpLQhhG+p2w1sQMxau+952hd5M+4r28dHej9hduBsfJx8eHfgo18Vfh6vG9eQ6VqMR3aZNVC1dSvW69Qi9HofAQLxuuB6PiRNx7t8f5RxDBxRU1vHfzWlsW7OebsV7SDAWo3J2I+HqmSRMvhxn9+aX+0LUVhmp05nwlfXnTSYDXZJamxCw7HHY9RX0uxmu+MA2o1AzpJSl8PG+j9mYsxFfJ1+eGvwUV8ddfXKaNmG1UrtzF5V/LKF61WqsVVWovbzwnD4Nz6lTcR448JwhDpBaUM3s9Uc4smUd/cv2MtpchZNvICNnPECP0ePQaM+ujmlNJTm2HqKyhUvTyUCXpNYkBCx/0hbmwx6ECa82qyVLVmUWnyZ+yoqsFbhr3Xl4wMPM7DbzZEcgQ0YGlYsWU7l4Meb8fFQuLrhfNh6PqVNxHTYMRXPuPxxCCLallzJ7/RGK925mUOU+xpir8Y6MZdQ1DxKbMKTV6sfPp1QGerPJQJek1iKEbezynV/C0AeaFebFtcV8mvgpv6f9jlat5e7ed3N7r9vx0HpgLi+nbOlvVC5aRN3+/aBS4TpyBAGP/Qv3Sy9F5Xz+FiFWq2DVoUI+X5OCNXUHg6r20dtUg39MHKOu+xdR/QbafXzx0twa3LwdcXLtfFPFtRYZ6JLUGoSAVc/B9s9gyH0w8fUmhXmduY7vD33P1/u/xmQ1cUO3G7ir9134ar3Rbd1KzsJfqV6zBkwmHLt3J+DJJ/G8fCoO/v5NKpbFKli2P59P/0zFIX0XQ6v24myqIbhrd4ZfO5PI3v3sHuQnlObWyPrzZmpSoCuKMgn4EFADXwsh3jxjuScwB4io3+c7Qoj/tnBZJaljEAJW/9vWnX/Q3TDpzfOGuRCC5ZnL+WDvB+Tr8hkXMY5HBz5KUJWKyq9+JO233zHn56P28sJn5o14XnUVTvENtzFviNliZXFSHp+sPYpDViKjqnbjYqgkuGt3Rl5/C+E9e7ebIAewmKyU59cS2Vu2cGmO8wa6oihq4FPgMiAH2KUoymIhxKFTVvsHcEgIcYWiKP5AqqIoc4UQxlYptSS1V0LYxmTZ+pFtIoop/zlvmCcVJ/H2rrdJLk6mu093XhvyEvEHKyl/+AXSt20HRcF15EgCn3wS90vHomi1TS6OxSr4bV8uH685gpJzmDHVu3GvLcYvMppRNz5CdL+EdhXkJ5QX6rBaBX6y/rxZmnKFPhhIE0JkACiKMg+YDpwa6AJwV2w/GW5AGWBu4bJKUvu34W3Y/D4MvN02u9A5wrJUX8p7e95jcfpi/J39eTP2nyTsKKfyzcfJLS1FExKC/8MP4XnllWiCg5tVDKtVsPxAAe+tTiW3oIRry1fjVZ2LV1AwI+56nPhho87b6sWeTt4QlVUuzdKUQA8Fjp/yOgcYcsY6nwCLgTzAHbheCGFtkRJKUkexby6sf902efPU9xudJs5itbDgyAI+3PchBmMtT9WNY/jGauq2vkOZouA2ZgzeN1yP64gRjXb6aYwQgnWpRbyz8giH8quIC3DjuXg9hatzGX/XA/QaO6HZIx7aQ0muDrWDCq8A2eW/OZrynW3oEkOc8XoikAhcCsQCqxVF2SSEqDptR4pyD3APQERE689ULkltJn0dLHnINibLtI8bDfMDJQd4dfurZB8/wO0ZYVyyywEKVmIOCMDvgQfwuubqZl+Nn7Ajo5S3VqSw91gFET4uvH99X6b1DWXV5+9T4+1D38umXMQHbFulOdX4hLiikhNBN0tTAj0HOLWPbxi2K/FT3QG8KYQQQJqiKJlAN2DnqSsJIWYDswESEhLO/KMgSR1T4SGYfyv4dbX1AG2g01CloZKP9n7Ero3zmZ6kYegBBZUxG5chQ/B+5jncL70U5QKvnNOKanhzeQp/Hi4kyMOJ12f05tqEMDT1YViclUFAZPRFfcS2VpKrI7Knj72L0eE05SdoFxCnKEo0kAvcAMw8Y51jwDhgk6IogUA8kNGSBZWkdqkq3zZqosYFZs4Hp7NHKFyTsYql373AqC2VXJMjUJwc8JxxNd43zcSpa9cLPnRxtYEP/jzCvF3HcdGoeWJSPLNGROOk+auaxmwyUZp7nOgBgy74OG2ttsqIvsooOxRdgPMGuhDCrCjKg8BKbM0WvxVCHFQU5b765V8ArwD/UxRlP7YqmieFECWtWG5Jsj9DDfx4HejLYdZy8Dp9sKqi4myWfvhPYlYf5u5KECGBBDx1O14zZjQ4NG1T6Y0Wvt6UwRcb0jGYrdw8JIKHxsXh63Z21/zS49lYLRYComIv+Hht7cSk0HIM9OZr0v94QohlwLIz3vvilOd5wISWLZoktWMWMyyYBYUH4MafIbjvyUWGnBz2fPISTss2M9QIld1DCXrlcbzGjW/2Tc5TCSFYnJTHm8tTyK+sY1LPIJ6YFE+Mf+PBV5Rt+0c5IKrjVLmcCHR5hd587f92tyS1RyuegqMrYep70NV2LaM/cJDcLz/B8Od63BVI6edD/4f+zdChEy/6cAdyK3lpyUF2ZZXTK9SDj27sz6Co89cxF2VmoHFyxivwwm602kNJTg0unlqc3Zve3l6ykYEuSc2V+OPJwbZEwix0W7ZQ+vXX1G7bjt5RYe1QDRGz7uOakfc1Ou1bU5XWGHhnVSrzdh3Hx0XLm1f15tqEcNRNnPChODsD/8jodt3m/ARhFRw7VEZOSrnsUHSBZKBLUnMU7Ic//omIGEm1aRilV19D3aFDKH6+LJ7oxep+8PG0b+nu2/2iDmO2WPlhezbvrT6C3mhh1ohoHhoXh6dz0weqElYrRVmZ9Bw97qLK0tpMBgup2/NJWptDRWEtLp5a+o2XzZovhAx0SWoqfQXix5upyPamdL0BU87jaKOjcX7uX/xD8wvl1mq+nvD1RYf53mPlPPfbAQ7lVzEqzo8XruhBl4Dmz9VZUVSAqU5PQFTMRZWntVSX1bF/XQ6HtuRhqDUTEOnOZbN6EDsgALVD+/+Poj2SgS5JTWCtraXimWso3ajHXKvGqZc3AU8+Sc2QHsxafSdVxmq+mvAVPXx7XPAxKmqNvLUilXm7jhHo7sTnNw1gUq+gCx5rpTjrxA3R9hXoRdlVJK4+RtreYgBi+vnTd1w4QTEe7XJcmY5EBroknYOlRkfFvJ8o/fIzLNV6nLuGE/zEy7iOGE5hbSF3rriDKkMVsyfMpqfvhU32LIRg4d5c3lh2mAq9iTtHRPPIZV1xc7y4X8+irAxUajW+YfavvhBWQfbBUhJXHyP3SAUaJzV9Lw2jz6XhuPs42bt4nYYMdElqgKWmhvI5cyn773+xVFbiGmTA77r+uDy2ABSFAl0Bs1bOosJQwezLZtPLr9cFHSe9uIZnft3PjswyBkR4MWdGb7oHt8ycnUVZGfiEhuPQjNEZW5rFZCV1ZwGJq49RXlCLm7cjw6/uQo+RITg6y/hpafKMStIpLDU6yufMORnkbqOG4ee1EedIP7j7O1AUhBA8tPYhyurKmH3ZbHr79272cUwWK19uSOejNWk4a9W8eVVvrksIR9XE1itNUZSVQWTvfi22v+Yw6s0c2JRL0prj1FYa8Qt3Y/wdPeiSEIBajs/SamSgSxL1QT53LmXffmsL8jFj8Lv/Xpx3PwGFerh+DjjamtJtz9/O4bLDvDz8Zfr492n2sZJzKnhiQTIpBdVM7R3MC9N6EODestUOuopydOVlbV5/XltlJHntcfZvyMWoNxPWzZvxt/UgrLu3rB9vAzLQpb81q15P+Y8/UvrV11gqKnAbPRq/B/+Bc+/esO51OL4DrvkW/P8ac+WHQz/g4+TDlJjmjV6oN1p4b3Uq32zOxN/dkdm3DGRCz6CW/kgAFGdnAuAf2TaBXlWiJ3H1MQ5tzcdithLbz5/+EyMJjGqZ6iOpaWSgS39LVqORil9+oeSLL7AUl+A6ciT+D/0fzn3qr7iP7YCN/4G+M6HX1Se3y6jMYFPuJh7o+wCO6rPHTmnMtvRSnlyYzLGyWm4cHMFTk7s1q015cxW1UQuXisJa9qzM5sj2AlAgfmgQ/S+LwDvItVWPKzVMBrr0tyLMZioXLaLk088w5eXhnDCQgPffxyUh4a+VDNXw693gGQaT3zpt+7mH5qJVabku/romHa/WaObtFan8b2sWkb4u/HT3UIbF+rbkR2pQUVYGHv4BOLm1To/L0twa9qzIJm13ISoHFb1Gh9J/QgRu3rLFij3JQJf+FoQQVK9cSfEHH2LMysKpVy+CXnoJ15Ejzq7bXf4UVB6H25eB019VBhV1FSxOX8zUmKn4Op8/lHdllfHYL0lkl9Zy+/AonpgUj4u2bX7lirIyWuXqvPhYNbuXZZGRWIzGUU2/yyLoNz4CFw857kp7IANd6vR027dT9M671B04gGNcF0I//gj38eMbvkl3aBEkzoFRj0HksNMWLTi6gDpLHTf3uPmcx6szWfjPylS+3ZJJqJdzm12Vn2Cqq6M8P5duwy9psX0WH6tm5x+ZZCWX4OjiwKCpUfS5NBwn19arNpKaTwa61GnVHTpE0bvvoduyBYfgYILfeAPPaVc0PoRtVT4seRhC+sOYp05bZLKa+CnlJ4YGD6Wrd+OTUuw7Vs6/5ieRUaLj5qERPD25O64X2UGouYqPZYEQLXKFXpRdxa6lWSeDfMi0aHqPDZdtyNsp+V2ROh3j8eMUf/gRVX/8gdrTk4Ann8R75o2oHM9xE9Nqhd/vB1MdXPXVWdPIrcpaRVFtES8Me6HBzU0WK5+sTeOTdWkEujsy584hjIzza8mP1WQnb4hGX3igyyDvmOR3R+o0LBUVlHzxJWVz56Ko1fjeey++d85C7dGEpnM7v4SMdbbxzf3iTlskhOCHQz8Q5RHFyNCRZ22aWaLjkZ8TSTpewVX9Q3lxek88nOxXFVGclYGTqxvuvv7N3rY0t4YdizPITJJB3hHJ75LU4VmNRsrn/kjJF19grarC8+qr8P+/h9AEBjRtB0UpsPoFiJsICbPOWryvaB8HSw/y/NDnTxvfXAjBTzuP88ofh9A6qPhkZn8u7xPSUh/rghVlpRMQHdOsjjzlBTp2/pFJ2p4itI5qBl0eTd9xMsg7GvndkjosIQTVK1ZQ9O57mHJycB05koDHH8MpPr7pO7FaYNE/QOsC0z+BBkLwh0M/4OnoyRWxV5x8r6TGwJMLklmTUsTILn68c21fgjzt32TParFQciybvhOa1umpsriWXUuzOLKjALVWzcCJkfS7LELe7OygZKBLHZI+MZGCN96gLikZx/h4wr/+GreRI5q/ox1fQu5umDEb3M6+os+pzmHt8bXM6jULZwdnADYcKeZf8xOpqjPz/OU9uGN4VIuOwXIxyvNzMZuMBESfe1JoXYWBXcuyOLw5D0Wt0GdcOAMmRMrmhx2cDHSpQzHl5VH07ntULV2Kg78/wa+9iueVV17Y5MtlmbD2FYibAH0a7ig09/BcVKi4If4GjGYr/1mZwlebMuka6Mbcu4YSH9T8iSdaU1FmOgABkQ1PCl2nM7F3RTbJ63MQFkGPkSEkTInC1avpvV6l9ksGutQhWHU6Sr/5htJvvgXA9/778LvrLlSuF9jFXAhY8hAoarj8/QarWnQmHb+l/cbE6InU6t2469ut7M+t5JahkTw7tTtOmgv4I9LKirIzUWs0eIeEnfa+sc5M8trj7Ft1DKPBQtfBgQy+PAZPf2c7lVRqDTLQpXZNWK1ULlpM8XvvYS4uxmPqVAL+9SiakIu8+bj3e8jcaGvV4hnW4CrLM5ejM+kIYhxTP9qERq3iy1sGMrGVBtRqCUWZ6fiFR6F2sP1qW8xWDm7KY/eyTPTVJqL6+DF0egy+chLmTkkGutRu6ZOSKHjtdeqSk3Hq04fQDz/EZUD/i99xVR6seg4iR8LAOxpdbX7qAlyVMD5YamBwtC8fXN+PEK/2e0UrhKAoO5O4QUMRVkHaniK2L0qnqqSOkDgvptwfS1CMp72LKbUiGehSu2MqKqL4vfep/P131P5+BL/5Bp7TpqGoWmBiBCFg6b/AYoRpH0Ej+1yWupvDZQcxFF7BP8fH8+ClXVC3kxufjakpK6WuugqNcxC/vLmb4mPV+Ia6cvmDfYno6SPHI/8bkIEutRtWo5Hy77+n5LPPESYTvnffhe+996F2a8GhWA/+CqnL4LJXwPfsliBCCH7ceYzXts9G7anh8+l3My6+4RuM7c3RXQcAOLDRiEeAkfG3dyducFC7aYEjtT4Z6FK7ULNxI4WvvY4xOxu3sWMJfOpJtJGRLXsQXSksewJCBsDQB85aXF1n4ulf9/PH/mw84xMZFzm+Q4R5dVkdW35J5vCmRQAMu3ow/cbH4tAOb9pKrUsGumRXxpwcCl9/g5q1a9FGRRH+1WzcRo1qnYOteg7qKmD6YlCf/qN/ILeSB3/cy7GyWq4YXsz6Mj0zuzdtzHN7MejNbP9tP4mrFmHWJwIW+k64goTJjQ8eJnVuMtAlu7DW1VH69TeUfvUVqNX4/+tRfG+7DaW1ZqjP2gxJP8LIRyGw58m3T3Tff3HJQXxctMy7ZxgfH36YKI8oBgYObJ2yXCSL2cqe5QfZ8fsCjDWJgJUug0Yy8oaZ+IaF27t4kh3JQJfalBCCmnXrKHz9DUw5OXhMmUzAE0+gCWrFpoBmI/zxKHhFwCWPn3y71mjmud8O8Ou+XEbF+fHB9f0oNx0nsTiRxxIea3c3EYUQHNyUysa5P6GvSAQEsQmjGH3zTLyDQ+1dPKkdkIEutRnj8eMUvvoaNRs2oO0SS8T//ofr0CGtf+Btn0BJKsycbxuzBUgvruGBOXs5UlTNP8d3PdmK5eudC3FQOZw2bkt7kJmUyZ9ff09V0V5AENV/FJfefjPeQcH2LprUjjQp0BVFmQR8CKiBr4UQbzawzhjgA0ADlAghRrdYKaUOzWow2Hp5fjkbRa0m4Ikn8LnlZhRNGwwAVZ4FG96GbpdD14kALE3O54kFSWgdVHx3x2Au6WobZtZgMbA4fTHjIsbh4+TT+mVrgvz046z8/DtKj+8EIKzHCCbeexterfkfjdRhnTfQFUVRA58ClwE5wC5FURYLIQ6dso4X8BkwSQhxTFGUJo5bKnV2NZu3UPDKy5iyj+E+eRKBTz2FJjCwbQ4uBCx/EhQVTH4Lo9nKG8sP898tWfSP8OLTmQNO6yi0Ons1VcYqrul6TduU7xxKc/NZ8fn/KDi6DYDALkOZdP8d+IXJK3KpcU25Qh8MpAkhMgAURZkHTAcOnbLOTOBXIcQxACFEUUsXVOpYTAUFFL75FtUrVqCNjCT8m69xG3EBoyFejJSlcGQFTHiVQsWPB77azp7scu4YEcXTk7ujdTi9U9HCIwsJcwtjcNDgti3nKaqKi1k1+zuykzcC4BM6iIn33UZI1wi7lUnqOJoS6KHA8VNe5wBnVnx2BTSKoqwH3IEPhRDfn7kjRVHuAe4BiIiQP6CdkTCbKZszh5KPPkZYLPg//BA+d96JqrVarzTGUAPLn4CAnuwMuJYHPtpMrdHMxzf254q+Z48Dk1WZxe7C3Tw84OHTJrFoKzVlpaz9bg5Hd6wFIXDz7c+4O2+ly8BzD4MrSadqSqA3dKtfNLCfgcA4wBnYpijKdiHEkdM2EmI2MBsgISHhzH1IHZw+KYn8F17EkJKC6+hLCHruObThdmpGt+FNqMplSdfXeOTbvUT4uPDj3UPoGtjwcLcLjy7EQXHgyi5XtmkxdRXlbJ73Ewc3rEJYrTi69WbE9TfSb3xPFNnDU2qmpgR6DnDqb2UYkNfAOiVCCB2gUxRlI9AXOILU6VkqKyl6/30qfp6Pg78/oR9+iPuEy+zX7K/wIGLbZ2zznMr/bdYyoUcA71zXt9F5Pk0WE4vTFzM6fDR+zm0zsXNtVSXbf/2FpFVLsVrMODj1pN/Eqxh+9UA0jrKHp3RhmhLou4A4RVGigVzgBmx15qdaBHyiKIoDoMVWJfN+SxZUan+EEFQtWULhW29jKS/H59Zb8Pu/h1p27JXmF4q63x+hDlceLJrGE5Piue+S2HOOZ7I+Zz1ldWVcHXd1qxdPX1PN7sW/smfZYiwmIyptN7oOu5wxNw3B3cf+U9hJHdt5A10IYVYU5UFgJbZmi98KIQ4qinJf/fIvhBCHFUVZASQDVmxNGw+0ZsEl+zJmZZH/0kvUbtuOU+/eRHw1G6cePexdLA6u/Iqe+Tt5S7mPj2aNZ2Tc+a+4F6UtIsA5gOEhw1utXIZaHXuW/s7uJb9jMuhRaeIJ7jaOsbeOIDhWDmkrtYwmtUMXQiwDlp3x3hdnvP4P8J+WK5rUHlmNRsq++YaSz79A0WoJ/PfzeF9//YVNAdeS5bIKvlyVyNXbXuOIQxx3/uN5wnzOP4lDib6Ezbmbua3nbahVLf8ZTHV17F2xhF2LFmKorUGl6YJH8CWMun4Y8UOCZD251KJkT1GpyWp37yb/hRcxpqfjPmkSgU8/jSbQ/l0OqutMPDo/iYQj7+HnUInnrQtwbEKYAyzNWIpFWJgeO71Fy2Q2GklavZydv8+ntqoStWMMTl5XMmDSIAZOjkTrJH/1pJYnf6qk87JUVFD4zjtULliIJiSEsC8+x33MGHsXC4C0ohru+WE3DqVH+dJxJUq/W3CMHNSkbYUQ/J72O338+hDjFdMi5bGYTRxY9yfbf51HTVkpGucotO6TiR3YhxHXdMHT36VFjiNJDZGBLjVKCEHVH0spfPNNLBUV+Nw5C/9//AOVS/sIpVUHC3h0fhKOaoU1kb+hKneF8S80efvDZYdJq0jj+aHPX3RZrFYLhzetZ9uCH6ksKsTJPQKN21j8wrsx8ro4Inr4XvQxJOl8ZKBLDTLm5FDw4kvoNm/GqU8fIr75Gqdu3exdLMBWX/7hmqN8uOYofcM8+e+QfLyWboEp74Br05sdLkpbhFalZWLUxAsui7BaObpzK1vmz6Us9zguXqE4us9A49aFkVfE0GtMKGp123dUkv6eZKBLpxFmM2XffUfxx5+gqFQEPvss3jNvtPtNzxNO1JevPlTI1QPCeO3yGJy+uBcCe59zwuczGS1GlmYu5dKIS/F0bH4rEyEEmYm72TJvDkVZ6bj5BOPqdyVmSzQ9RocwdHosLh5t3DtW+tuTgS6dpN9/gPx//xvD4cO4jRtH0HPPogluP4NBZRTXcM8Pe8gs0fHCFT24fXgUyrrXoCoHrv76rFmIzmVjzkYqDZVMi53W7HLkHDrApnnfk5d6CDcff3wirkRXFUVQlBeX3NCVgEiPZu9TklqCDHQJq05H8UcfUfbDHBz8/Aj9+CM8LrvM3sU6zbqUIh6atw8HlcIPdw5meKwflGXAlg+hz/UQOaxZ+1uUtgh/Z3+GhTR9u8KMNDbP+56spL24eHoT1msGxTkRaBVnxt8RS/xg2QxRsi8Z6H9zNRs2kP/SS5jz8vG68QYCHn0UtXvD453YgxCCzzek85+VqXQP8uDLWwYS7lN/U3bFM6DWwmUvN2ufJfoSNuVu4taet+KgOv+vQGnOMbb8PIejO7fi5OZO3JCrKMiOoKxAzYAJ4SRMiULrLH+VJPuTP4VAnclCqc5IWY2REp2B0hojvq5axnazfxvr1mIuKaHw9depWrYcbZdYIn+ci8uAAfYu1mlqjWYeX5DM0uR8rugbwttX98FZW1+Xf2QVHFluC3P35k32sCxjWZPanlcWFbD1lx85vGk9GidHeo6ZQWlBHMePmIno4cPI6+LwDrLjMAeSdIZOGegGs4UynZHSGqPtsT6kT4R2qc5AyYllNQZ0RstZ+1ApcPClSX8FSCchhKBy4UIK3/4PQq/H76H/w++uu1pvcuYLdLyslnt+2ENKQRVPTe7GvZfE/DXYl9kIK54C3y4w5P5m7VcIwe/pv9PbrzexXg0PTVtTXsaO334m+c+VqFQqeo+bisHQj/QkHR5+Dky+rzvRff3a3ZyjktThAj2vQs+urLLTwrqkxhbMJ0K82mBucFuNWsHHVYuvqyO+blqifF3wqX/u56Y9+XxHRhlvrUihpMbw17/3nYAxK4v8f79A7c6dOCcMJPjll3GMaZkONS1pW3opD8zdg9kq+Pb2QYyNP+M/pR2fQ1k63LQQHJr3hyilLIWj5Ud5bshzZy3T11Sza/FC9i1fgtVipueYy3D1HsGBjZWAnsFXRNP/sggcOtkfeanz6HCBnni8gofnJQKgVp0IaC2+blp6e3vZnrtq8XVzxMf1RFDbXns4OTTpqqpKbwKgqLquUwS6MJko/fa/lHz6KYqjI0Evv4TXNdegqNpX+2ghBN9tzeKVpYeJ9nNl9i0DifE/owt/dYFtjtCukyFufLOPsTh9MRqVhknRk06+Z6zTs3fZYnYv+RWDvpZuI0YT0XsyiX9WUrW3nNgB/gy/ugsevs7n2LMk2V+HC/SRcX78+ehofF21eDprzjks6oXyd3cEoLja0OL7bmv65GTyn/83htRU3CdOJPDZZ9AEtL97Awazhed/P8D83TmM7x7A+9f3w72h8cv/fBEsRpj4WrOPYbKYWJqxlLHhY/F09MRsNJL853K2/zYffVUlsQlD6TP+Gg5uNrBxXiHeQS5Me6Qf4d3ax4TRknQ+HS7QPZw0jU5U0FIC3G3jUhd14EC36nQUffgh5T/MwSEggLDPPsX90kvtXawGFVXVce+cPew7VsFDl3bhkfFdG/5DfXwnJP0EIx8F3+ZPzbYxdyPlhnKmRV/B/rWr2LbgJ6pLiwnv2YehV91E7lEnVn5zDLWDihHXdKH32DDZy1PqUDpcoLcFH1ctKqXjXqGfbIqYX4D3jTfg/+ijqN2aNvpgW9t3rJz75uyhus7M5zcNYHLvRjoyWa2w7HFwD4ZR/7qgYy06+ju9SwNJe/9HKvLzCIqNY8J9D2EyhLB2Thq6CgPdhgYxdEYsrp6OF/GpJMk+ZKA3QK1S8HNzpKiqYwX6WU0R587FZUB/exerUQv25PDMr/sJ9HTk1weG0y3oHD0sE+dAfiJc9TU4Nu+PkxCC5J3rcZ+XSmSVE+owB6Y99ixeQb3YPP8oeUcP4hfuxsS7e8nJJqQOTQZ6I/zdHSmu6RiBLoSg8tdfbU0Ra2vbbVPEE8wWK68vS+HbLZkMj/Xl05kD8HY9R1n1FfDnSxA+FHpf06xj5Rw+wOZ535ObcgiNs0L/22cydORV7F52jNXf7sLRRcPomfH0GBnSKvdjJKktyUBvRIC7I0XVdfYuxnkZs7LIf+FFanfsaNdNEU8o1xl58Ke9bEkrZdaIaJ6Z0g2H89VTb3gLakvhll+hiW2/CzPS2PzzD2Ql7sHV24fMwRrKYt24zmMMP720C4PORM9LQhkyLQYn19a9JyNJbUUGeiP83R05lF9l72I0SphMlH7zLSWffWZrivjii3hdd227a4p4qsP5Vdzzw24Kqwy8c21frhkYdv6NilJgx5cw8HYI7nve1Utzj7N1/lyObN+Mk5s7l9x0By6D4/hlwTNcu+8+1hWmENzFk1HXd8U/vP0McSBJLUEGeiMC3J0oqTFisQrU7exfcX1Skq0p4pEj7bop4qmWJufz2C9JeDg7MP/eYfQL9zr/RkLA8sfB0R0uPfckFFXFRWxd8COHNqzFwdGRoVffQMLlM7CYNcz+6jeuOvooWg8NI+6Io+vgQNnLU+qUZKA3wt/dEYtVUF5rxM+tfbR4sNToKP7gA8rnzsUhMLBdN0U8wWIVvLc6lU/XpTMw0pvPbx5wslnoeR1aBJkb6yeuaHjGH11FOdt//ZnkP1egqBQGTJnG4CuvxdHVnf3rctj5RyYqgw/l3dO4+95Zci5PqVOTP92NCKjvXFRUZWgXgV69di0FL7+CubAQ75tuwv+Rh9ttU8QTKvUmHpm3j3Wpxdw4OJwXp/XE0aGJ3eaNOlj5LAT1hoRZZy3W11Sze/FC9i5fgsVsovfYCQy9+gbcff04friMTT/vpLygFtdY+N79DV6//EUZ5lKnJ3/CG3Gyt6idW7qYCgspfPU1qlevxrFrV8I+eB/nfv3sWqamSCuq5p7v93CsrJZXr+zFzUMjm7eDze//NXGF6q8/AkZ9LXuWLWL3kt8w1unpNvwShl93E95BIVQW61n2eTKZSSV4+Dsz9YE+vF/8Cg4lVoaHDG/hTyhJ7Y8M9Eac7C1aZZ+WLsJqpXzePIrffQ9hNuP/6KP43nE7iqb9t8g4MXmzk0bFj3cPZXB0M7vOn5i4ovd1JyeuMBkNJK1cys5FC9BXVxGbMJQR19+Mf0QUJoOFHYsz2LfqGIpaYeiVMfQdF06VpZKNiRu5ucfNTRr3XJI6OvlT3gh7XqHXpR6h4N//Rp+UhOvw4QS9+ALaiIg2L0dzWa2CD9Yc5aM1R+kT5skXNw8kxOsCBrQ6ZeIKi9nEgXWr2b5wHjXlZUT26c+I628muEs8QgiO7ipk669p1JQb6Do4kGEzuuDmbfveLU9bjlmYL2iaOUnqiGSgN8JZq8bd0aFNe4ta9XpKPv+C0m+/Re3uTsjbb+FxxRUdokVGVZ2Jf85LZE1KEdcMDOPVK3vhpLmAYWbrJ66wjnuRQ3sPsm3BT1QVFxIS34MpDz1OeI/eABQfr67v5VmBX7gbl93Zk5AuXqftalHaInr49iDOO64FPqEktX8y0M+hLXuL1mzaTMHLL2M6fhzPGTMIeOJxHLy92+TYF+vU+vKXp/fklqGRF/ZHyGxALH+SVGtvti5MoTx/DYExXRh/5/1E9RuIoijoq41sX5zBoc15OJ2jl2dqWSqHyw7z9OCnW+hTSlL7JwP9HPzdHSlu5St0c3ExhW++RdXSpWijo4n47jtchwxu1WO2pJUHC/hXfX353LuGMCSm4eaF5yOEIH3ui2zZ40WJwRW/cNt4K10ShqIoChaLlQPrbc0QTQYLfcaGMWhqdKO9PBelL8JB5cCU6CkX8/EkqUORgX4O/u6OHMxrnd6iwmql4pcFFL37rm0quP97EN+770bVTsdfOdOp7cv7hnny+QXWlwshyErcw5af/kthdjbebu5MvfdfxA8bdbLX67FDpWyef5TyglrCe/gw8po4fEIan8vTZP1r3HMvJ68L/YiS1OHIQD+HAHcn1lUVtfh+61JTKXjhRfSJibgMGULQCy/gGBPd4sdpLeU6Iw/N28emoyVcnxDOS9N7Nru+XAjBsf1JbPllDvlHUvBwUTExNJMezyxB5Wcbi6aisJYtC46Stb8UD39npjzQh6jevuetztmcs5myurLzTgItSZ2NDPRz8Hd3RGe0oDOYcXW8+FNl1eko/vQzyr77DrWHB8FvvIHnldM7xE3PEw7kVnLfnD0UVRl446re3Di4+a1vcg4dYMv8OeQcPoC7rz+XTR9Hz9SXUI97DvxiMNSa2LU0i/3rclBrVQy7Kpa+Y8NRa5o2Ts3vab/j4+TD8FDZ9lz6e2lSSimKMgn4EFADXwsh3mxkvUHAduB6IcSCFiulnQScMhXdxQZ69Zo1FLz6Gub8fLyuvZaAfz2K2surBUrZdhbsyeHZ3/bj46pl/n1NHI/lFLkph9j6y1yOHUjC1duHS++4l96XjMbhq1Hg1wXr0Ac5tDGXHYszqNOZ6DE8mCHTY3HxaHo1VHFtMRtyNnBrz1vRqNp/m31JaknnTSlFUdTAp8BlQA6wS1GUxUKIQw2s9xawsjUKag+ntkWP8mu8zvZcTLm5FLz2OjVr1+LYtSuh777briedaIjBbOGVPw4xZ/sxhsX48vHM/s0aDiHvyGG2/vIj2cn7cPH0YvQtd9J3whQ0WkdY9zqUZ3F89CK2vJVMaW4NIXFejLw2Dv+I5o+GuCh9ERZh4eq4q5u9rSR1dE257BwMpAkhMgAURZkHTAcOnbHe/wELgUEtWkI7CvD4azyX5hJGI6X/+46Szz4DlYqAJ57A55abO0RPz1PlVuh5YO5eko5XcM8lMTwxMf7845fXy09LZesvP5KVuAdnD08uuXkW/S6bgsapfnCu0nTK181jq/Vjsn624u5rZtI9vYjp739B1VBWYWXBkQUMChpEpEczhxqQpE6gKYEeChw/5XUOMOTUFRRFCQVmAJdyjkBXFOUe4B6AiA7Q89Hf7USVS/O6/+u2b6fg5VcwZmTgftllBD79FJqQkNYoYqvaeKSYh+ftw2QR557v8wwFaUfYtvAnMvbuwsndg1Ezb6ffxKlonf5qBVNXY2TXJ4s4UPQOakctw2ZE0+fSMBwupDNSvR35O8ityeWh/g9d8D4kqSNrSqA3dKkkznj9AfCkEMJyrisrIcRsYDZAQkLCmftod7xdtDioFIqaOFm0qaiIorfepmrpUjQREYTP/hK3Sy5p5VK2PKtV8PHaND5Yc4SuAe58fvMAYvzPP7Jj/tFUti34kczEPTi5uTPyhlvpP+lytM4uJ9exmK0c2JDLrsWpGA296RFfw+A7xzSrnrwxC48uxNPRk3GR4y56X5LUETUl0HOA8FNehwF5Z6yTAMyrD3M/YIqiKGYhxO8tUUh7UdVPFl18nkAXZjPlc+dS/NHHCJMJvwcfxPfuu1A52n/Y3eYq1xn55/xE1qcWM6N/KK/N6IWL9tw/JnlHUti28CeyEvfg5O7ByBtvo//EqacFuRCCjMRitv2aTmWxnnDnFEZ0W4/vQz+fNprihSqrK2PNsTXcEH8DjuqOd94lqSU0JdB3AXGKokQDucANwMxTVxBCnGxErSjK/4A/OnqYnxDg4XjOK3Tdzp0UvvIqhqNHcR01iqDnnkUb2THrb5OOV/DA3L0UVxt49cpe3DQk4px12bkph9j+6zyykvbifKJqZcKU04IcoDCzii0Lj5KfVol3sCtTh+4jMvMVlGvWtEiYAyxOW4zZauaars2bRFqSOpPzBroQwqwoyoPYWq+ogW+FEAcVRbmvfvkXrVxGu/J3cyS/8uw6dFNhIUVv/8dWvRISQtgnH+M2blyHalN+ghCC/23N4vVlhwlwd+KX+4bR9xxNEo8f2s/2hT9x7EAyzh6eDdaRA1SV6Nn+ezpHdxfh7KFlzE3xdI8pQfX1qzDoDggb2GLlX3h0If0D+hPrFdsi+5SkjqhJjauFEMuAZWe812CQCyFuv/hitR8BHo4k5VSefC2MRsp++IGSTz9DmM34PfCArXrF+QKGiW0HKvUmnlyQzIqDBYzvHsA71/bFy+Xs+uwTPTu3LfyJ3JSDuHp525ofjp/8V6uVenU6E3tWZJO87jgqRSFhShT9J0Sg1QDf3ATOPjDu3y32GXYX7iarKou7+9zdYvuUpI5I9hQ9D383R8p0BixWgX7rVgpffx1jRgZuY8cS+MzTaMPDz7+Tdmp/TiX/+HEveRV6npvanTtHRp/1H4YQgsx9u9n+6zzyj6bi5uPL2Nvvpfe4CbZ25KcwmyzsX5fLnhVZGPRmug0NYsi02JPjk7P1Y8jbB9d8C84tN5LkgiMLcNe4c1nkZS22T0nqiGSgn4e/hxOBNSVk3nc/po0b0EREEPbF57iPGWPvol0wIQQ/bM/m1T8O4+em5ed7hzEw8vSAtVotHN2xjR2/z6c4KwMP/wDG3/UAPcdchsMZbemFVXBkZwHbF2dQU2Ygoqcvw2bE4hd2SsuYsgxY+xp0nQw9r2qxz1JpqOTP7D+5Ku4qnB065n9JktRSZKCfg1WnI27R93y55meMjo4E/OtRfG67rcOMiNiQyloTT/+WzLL9BVzaLYB3r+2Lt+tfn8diNpOyZQM7f/+FsrwcvINDmfTAP+k2YjRqh9N/XIQQHD9cxrbf0ik5XoN/hDvjbu1OWLczppwTApY8DGoNTH0XWvA+w5L0JRitRnkzVJKQgd4gIQRVS5ZQ9M67eBYV8Wf4QPq89Aw9hvewd9Euyp7sMh76KZHCqjqentyNu0fFnJwYwmw0cmD9n+xespDKokL8I6K4/JEniRsyHFUDLVEKM6vY9ns6uanluPs6cdmdPYgbGIiiaiCs9/0AmRvh8vfBM7TFPo8QggVHFtDbrzfxPvEttl9J6qhkoJ9Bn5hI4Rtvok9KwqlXLzSvvs27y0p4W9P8cUXaC4tV8Pn6NN7/8yihXs4suH/4yYG1DLU6ElctY++yRdRWVhDcJZ6xt99LzIBBDbbYKS/QsWNRBun7inF21zDyujh6jQptfCTEqnxY+RxEjoABt7fo50oqTiK9Mp2Xhr/UovuVpI5KBno9U34+Re++R9Uff6D29yP4tdfwnHElBouAZSsoamb3//aisKqOR+Ylsi2jlGl9Q3h1Ri88nDToKsrZu3wxiSuXYtTXEtmnP0OuvJawHr0bDPKa8jp2/pFJytZ8HLRqBl0eTb/x4WidzvMjtOwxsBjgio9A1bQxYJrqlyO/4OLgwqSoSS26X0nqqP72gW7V6Sj95htKv/0vCIHvfffid/fdqFxtoys6qcDDyeG8vUXbo7UphTz2SzJ6o4W3r+nDtQPDqCws4M+lv3Nw3WrMZhNdh4xg8PRrCIzp0uA+9NVG9qzM5sCGXIQQ9B4bRsLkKJzdm3Af4dAiSPkDxr8Ifg3v/0KV15WzMmsl02Kn4aJxOf8GkvQ38LcNdGGxULl4CcXvv4+5qAiPqVMJ+NejDQ6iFeDh1OTxXNoDvdHC68sO88P2bLoFufPJzAG41xTwx4dvc3T7FlRqFd1HXcqgaVfjE9Jwnbah1kTin8dJWnMcs9FC/JAgBl0ejYdfE1uS6Mth6WMQ1AeG/V8Lfjqb+anzMVgM3NT9phbftyR1VH/LQK/ZsoWi/7yDISUFpz59CP3gg3OOUe7fhPFc2osDuZU88nMiaUU13DUiiutC9ez78g2OHUhC6+xCwrSrGDDpCtx8Gp7M2WSwkLzuOPtWHcNQayZ2QABDpkXjHdTM8eBXPgu1pXDzAlC37I+Z0WJkXuo8RoSMkD1DJekUf6tAr0s9QtE776DbtAlNaCgh776Dx+TJJycjbkyAhyP7jlW0TSEvkMUq+GpTBu+uSsXPWc37A8xUb/qCJdmZuHn7cMnNs+gzbhKOLg1XT5iNFg5uymPPymz0VUYie/sy5IqYC5pkgsN/QOJcGPUYBPe9yE92thVZKyjRl/DqiFdbfN+S1JH9LQLdVFhE8ccfUfnrb6jc3Ah44gm8b76pye3JT1yhCyHa5VgteRV6Hp2fyN6j+VzjepyInD1k7C/DNyyCifc9TPdRY1A7NDyxhtlk4dDmPPasyKa20khovBdD7u1NcKznhRWmuhCWPGQL8tFPXsSnapgQgh8O/UCsZyzDQ+ScoZJ0qk4d6Jbqakq//oay775DWCz43HILfvff1+y5PAM8HNGbLNQYzLg7tZ8Zh4QQ/LYvl/8s2Ea3skTuq0lBmAz49epLwn0PE9V3QKN/gCwmK4e35rF7eTa6CgMhcV5MmNWT0PiL6JIvBCz+PzDqYMZscGj5Dli7C3eTUpbCi8NebJd/XCXJnjploFsNBsrn/kjpl19iqazEY8oU/B95GO0FzpLkf8pk0e0l0Iuq6njt22UYktZzdW0mKpWKbiMuIeHyGQRExTS6ncVk5fC2fPasyKKmzEBwrCfjb+9OaLz3xQfknv/B0ZUw6S0I6HZx+2rE94e+x9vRm6kxU1tl/5LUkXWqQBcWC5WLFlP88ceY8/NxHTEC/0f/iXPPnhe13wB322iCRdWGJs3c05osZhPzf1nK/pWLidAXgeNfNzrdff0a3c5ssnB4Sz57V2ZTU24gMNqDsTd3I7y7T8tc6Zamw8pnIGYsDL7n4vfXgOyqbDYc38A9fe7BycHp/BtI0t9Mpwh0IQQ1a9ZQ/OGHGI6m4dSrFyGvv4brsGEtsv9Tr9Dtpbaqkp0rlrJj6RIc6qpxdvah97WzGHv5lLOGrz2VyWjh0KY89q6y1ZEHd/Hk0lu6E9a9Ba7IT7CY4dd7QK2FKz9r8Q5EJ8w5NAcHlQM3dLuhVfYvSR1dhw50IQS6zVso/vBD6g4cQBsZSegH7+M+cWKL1q8G1Ae6PdqiF2ams2/FEg5tWo+wmMl3DiNq0g08efNUtJrGv33GOjMHN+ax789j6KtsNzsnzOpJSFevlq973vwe5O62DYvr0TqTYVcaKlmUvojJ0ZPxc278PxFJ+jvrsIFeu3s3RR98gH73HjQhIbau+tOnoTi0/EfydNagVavarPu/xWwmbdc29q1YQm7KIaxqDQddulIdM5SXbhtHr9DGW6DU1ZhIXnec5HU5GGrNhHf3JuHuXoTEebVOYXP3wPo3ofe10Ovq1jkGtgmg9WY9t/a4tdWOIUkdXYcL9LrUIxT95z/oNm9G7e9H4PPP4XXtta06pK2iKPi7t37nIl1FOfvXrCTpz+XUlJXi4OXHnsCRJLl05d7LenPf6Fi0Dg1XZ+gqDCT+eYwDm/IwGyzE9PNnwKRIAqM8Wq/AhmpbVYt7EEx5p9UOY7Ka+PHwjwwJGiJHVZSkc+hwgW4pK6XuwAECHn8c75k3ttnUb36tFOhCCHIPHyRx1VKO7tyK1WIhqHsfDkZdxpIyT/pG+PDbNX3oGthwB5+Kwlr2/XmMlG35CCvEDQpgwMRIfENa+ebtiTHOyzLg1sXg7NVqh/oz+08Kawt5fujzrXYMSeoMOlyguw4bRpe1a1A10uOxtQS4O3K8rLbF9mfU13Jo03qSVi2l5Hg2jq6u9J14Odn+fXhzRwWmGivPXh7PHSOiUTcwxnhBZiX7Vh0jI7EYtVpF92HB9J8Qiad/G83as+trOLAQxr0A0aNa7TAnOhJFeUQxKqz1jiNJnUGHC3SgzcMcbC1d9mSXX/R+CjPSSP5zBYe3bMBUpycgKpYJ9z6EiOnH80uPknS4hBFdfHljRh8ifE//nMIqyD5Yyr5Vx8g7WoGjiwMDJ0bSe2wYrp6OjRyxFeTsgRVPQ9dJMOKRVj3UjoId7C/Zz3NDnkOltE7rGUnqLDpkoNtDgLsjZTojJosVjbp5wWKs05OyZSPJf66gMOMoDlpH4oeNos/4SXhExPLBn0f57+zdeLto+OD6fkzvF3JaSxSzycKRnYUkrTlOWZ4ON29HRl4bR/cRwecfj7yl1ZbBL7eBRzBc+XmrNVEE29X5x/s+Jsg1iBlxM1rtOJLUWchAb6ITbdFLagwEezatWqMwI439a1dxePM6jHo9vmERjL39XnpcMhYnVzdWHSzgxfc3kldZx42DI3hqUjc8Xf7qiVpbZeTAxlwObMhBX23CN9SN8Xf0oEtCAOpm/lFpEVar7SZoTSHMWgkuPuff5iJszNlIcnEyLwx7Aa26487jKkltRQZ6E53oLVpcfe5Ar6up4fCW9exfu4rirAwcNFriho6g7/jJhMR3R1EUskt1vLJgF38eLiI+0J2FM/szMPKvcCzL05G05hipOwqxmK1E9val37jwlumefzE2vwtpq2HqexA6oFUPZRVWPkn8hHD3cKZ3md6qx5KkzkIGehOduEIvqjq7pYuwWsk5fID9a1dxdMdWzCYj/lExjJt1P91GjsbJ1dbipNZo5rN16czelIFGpfD05G7MGhmNRq2y1Y8fKCV5fQ7HD5Wh1qjoNiyIvuPCmz8WeWvI2ADrXofe10HCrFY/3Ors1aSUpfD6yNfRqNrH+DmS1N7JQG+iE71Fi2v+CvTKokIObljDoY1rqCwqxNHFlZ5jxtP70gmnTekmhGDp/nxeX3qYvMo6ZvQP5anJ3Qj0cLLNDLQ1h/3rc6gqqcPVy5Eh02LoeUkIzm7tpJqhMgcW3gl+XeHy96GV/0uwWC18mvgpsZ6xTIme0qrHkqTORAZ6E/m52QK9sLSKgxvWcHDDGo4fTAZFIaJnH4ZfdzNxg4ehcTx9XJWUgipeXHyQ7Rll9Aj24KMb+5MQ5UNpXg0b/kglZUcBZoOF4C6eDL0ylpj+/vapH29MXRXMvQ7MBrjue3Bs/cHJlmYuJbMyk/fGvIdapW7140lSZyEDvQmsVgv5h/czpXw95jnprDAb8QoMZsR1N9Pjkkvx8A84a5ui6jreX32En3cdx8NZw6tX9uK6AWFkJ5Xw28K95B2tQO2gIm5wIH3GhF3YzECtzWKGBXdAcYptKjn/1u+labKa+CzxM7r7dGdcxLhWP54kdSYy0M+h5FgWhzat4/Dm9dSUlRKh1lIZ3JN777qe0G49G7xBqTda+HpTBl9sSMdgtnLb8Chm9QsnZ08xc57dir7ahIefE8OuiqX78OD2U61yJiFgxZOQ9idc8SHEXtomh/3t6G/k1uTyzLhnZLtzSWomGehnqC4tIWXrRg5vXk9xVgYqtZqovgMYc+tdvJqsUGNRCOve66ztrFbb7EHvrEolv7KOiT0CuCM6iJLkMhYv240CRPb2o9foUCK6+6A00PuzXdn+ua036PCHYODtbXJIg8XAl8lf0te/L6NCZa9QSWquJgW6oiiTgA8BNfC1EOLNM5bfBJyYQLIGuF8IkdSSBW1N+uoqjmzfQsqWDeSkHAQhCIqNY+zt99JtxCW4eNhGN/Q7lkhGZtlp2woh2HS0hLdXpnAgt4ohAR481i2QqoMV7Nl6FBcPLQMnRdJzVCjuPh1kUoaUpbbJKrpPg/Evtdlhf0n9haLaIl4f+bqcXk6SLsB5A11RFDXwKXAZkAPsUhRlsRDi0CmrZQKjhRDliqJMBmYDQ1qjwC3FUFtLxp4dpGzdSFbSXqwWCz4hYQy/dibdRozGO+jscb0D3B0prvlrsug92eW8vSKF3RllDNY685yrL4YjtRQqRUT08qXHiBCievuiak83Oc8nbx8svMvWznzGl63aE/RUtaZavtr/FUOChjAkuF3/6EhSu9WUK/TBQJoQIgNAUZR5wHTgZKALIbaesv52IKwlC9lSjPpa0vfu4si2TWQm7sFiMuHm68eAKdPpPnIM/pHR57wy9Hd3xGi2sjOzjK82prP/QAmDhCOPmlyh0orWx0rfK6LpPjwYN+8OcjV+qorj8OMN4OILN/wE2rYbM2d28mzK6sp4sP+DbXZMSepsmhLoocDxU17ncO6r7zuB5Q0tUBTlHuAegIgLnLC5uYz6WjL27ebIts1k7tuN2WTEzduHvuMn03XYKELi4lGaeBXq7+6IixX+89Fu+pgd6G92Qq1REdPPn+7Dggnt5o2qvdeNN6YqH767Akx6uOU3cA9ss0OnlqXy3cHvmB47nX4B/drsuJLU2TQl0BtKKNHgiooyFlugj2xouRBiNrbqGBISEhrcR0uoq6khfc8OjuzYQnbyPiwmE65e3vS6dALxw0cR2rV7k0McbPNyZiWVYNhUyP1VzqgAv0h3eo4IIS4hAEeXDt6TsabIFua6Yrjldwjs0WaHtlgtvLztZdy17jyW8FibHVeSOqOmBHoOEH7K6zAg78yVFEXpA3wNTBZClLZM8ZpOV1FO+m5biB8/mIzVYsHd15++4ycTN2Q4IfHdUTWjk4rVKshNLefIjgLS9xVjMlhw83YkYWIE8UOD8QluB93xW4KuBL6bBlW5cPOvED6oTQ8//8h8kkuSeX3k63g5ebXpsSWps2lKoO8C4hRFiQZygRuAmaeuoChKBPArcIsQ4kiLl7IRpbnHSd+9g7Rd28hPOwJC4BUYzMCpVxI3ZDhBsV2b1VpCCEFRdjVHdxeStqsQXaURrZOaLgkBxA8JIqSLV/tvbtgctWXw/ZVQngkz50PksDY9fKGukA/3fsiw4GFcHnN5mx5bkjqj8wa6EMKsKMqDwEpszRa/FUIcVBTlvvrlXwD/BnyBz+oD1CyESGiNApfl5XJg3SrSdu+gPC8HgMCYLoy49iZiBw3FLzyy2SFemquzhfjuQqpK6lA5KET08GXkkCCi+vjioOmE3c/rKmHOVVCSCjf+BDGj27wIb+x8A7PVzPNDn5fNFCWpBTSpHboQYhmw7Iz3vjjl+V3AXS1btIaV5+ewZ+nvhPfsw4BJVxCbMAR3X79m7UMIQVm+jvS9xaTtLqS8oBZFpRDWzZuEKVHE9PPv+PXi52KohjlXQ8EBuH4OdBnf5kVYc2wNa46t4ZEBjxDuEX7+DSRJOi9FiFa7N3lOCQkJYvfu3c3ezmI2YTIYTg5J21RCCEpyakjfW0T63mIqCmtBgeBYT7oOCiSmfwAuHu20G35L0pXAj9dBXiJc9x10v6LNi1BjrGH6oul4OXox7/J5cnhcSWoGRVH2NFYD0uG6/qsdNKgdmhYAwioozKoiI7GY9H3FVBXrURQI6epNn7FhxPT3b9u5OO2tLNN2ZV6VC9f/AN2m2qUYH+/7mOLaYt4f874Mc0lqQR0u0M/HbLKQk1JOZlIJmckl6KuMqFQKod28GTAhgph+/ji7/w2uxM+Utw/mXgtWM9y2BMIH26UYycXJ/JTyEzd0u4E+/n3sUgZJ6qw6RaDXVhk5drCUrP0lZB8sw2ywoHFUE9nLl+i+fkT28u3cdeLnk/Yn/HyrrQfoLb+CX5xdilFeV87jGx4n0DWQh/o/ZJcySFJn1iED3dYypYasZFuIF2ZVgQAXDy1dBwcS09efsHhv1JoONIZKa0n8CRY/CAHd4aYF4B5kl2KYrWYe3/g4JfoSvpv8HW7a1p8oQ5L+bjpcoGftL2HDj6nUlNumgguIdGfw5dFE9vLFP9y9c7UTvxhWK2x6B9a9BjFj4LofwMnDbsX5aO9H7MjfwcvDX6aX39nDD0uSdPE6XKC7eTsSEOXB4Ct8iejp+/e6qdlUtWXw231wdCX0uR6mfQIO9rtvsCJzBf89+F+uj7+eGXEz7FYOSersOlyg+4W5M/ne3vYuRvuVuxfm3wbV+TDlHRh0V6tP6nwuqWWp/Hvrv+kf0J8nBz15/g0kSbpgHS7QpUYIAbu/gRVPg1sgzFoJYQPtWqRKQyWPrHsEN40b745+F436b3xjWpLagAz0zsBQA0sehgMLIG6CbWIKFx+7FslitfDkpicpqC3gvxP/i7+Lv13LI0l/BzLQO7rjO+H3B6AsHS59HkY+2mazDDVGCMEHez9gS+4Wnh/6vBzjXJLaiAz0jspQA2tfhR1fgEeobRxzOwywdSYhBO/veZ//Hfwf18dfz7Vdr7V3kSTpb0MGekeUvtZWxVJxDAbdDeNfAEd3e5cKIQRv7nyTH1N+5Ib4G3h6yNNyFEVJakMy0DsSfTmseg72zQHfLnDHcogcbu9SAWAVVl7Z/goLjizg1h638ljCYzLMJamNyUDvKAoPwg9X2aaJG/kojH4SNO1jImqL1cK/t/6bxemLuav3XTzU/yEZ5pJkBzLQO4LyLFuYKyq4ey2E9LN3iU4yWU08u+lZlmct54F+D3Bfn/tkmEuSnchAb+9qiuCHGWCug1krbGOytBM1xhqe2fwM646v45EBj3Bn7zvtXSRJ+luTgd6e1VXZxi+vLoBbF7WrMD9YcpDHNz5Obk0uTw1+ipu632TvIknS354M9PbKVAfzZkLRIbhxnt3GLz+TVVj54dAPfLD3A/yc/fjvxP8yIHCAvYslSRIy0NsnqwV+vQuyNsFVX0HcZfYuEQBldWU8u/lZNudu5tLwS3l5xMt4Onrau1iSJNWTgd7eCAF//BMOL4FJb0Kf6+xdIgB25u/kqU1PUWmo5Jkhz3BD/A3y5qcktTMy0NsLQzUcWAh7voO8vTDqMRh6v71LRW5NLh/v+5ilGUuJ8oji8/GfE+8Tb+9iSZLUABno9iQE5O6BPf+DA7+CSQf+3WHqe5Awy65FqzRU8lXyV/yY8iMqRcWdve7knj734KJxsWu5JElqnAx0eyjLhJQ/bNPDFR0EjQv0ugoG3A5hCXYdv9xgMfDT4Z+YvX82NcYapneZzj/6/YMgV/tMXSdJUtPJQG8LQthaqxz+w1Y3Xrjf9n5If7j8feh1jV2nhwOoqKtgcfpi5hyeQ74un1Gho3hk4CN09e5q13JJktR0MtBbi6EasrdB5gZIXQZlGYACEUNhwmvQ/XLwjrJrEYUQ7C3ayy9HfmF11mqMViP9/PvxyohXGBI8xK5lkySp+WSgtxSTHo7vgMyNtq/cvSAsoHaEqBEw/P8gfiq4B9q7pJTqS1meuZxfjvxCRmUGbho3roq7imvjr5VX5JLUgclAvxAWMxQftoV23j5bq5TCQ2A1gaKG0IEw8p8QfYmtQ5DG2a7FtQorh0sPszFnIxtzNnKw9CACQR//Prw8/GUmRk2UNzslqROQgX4uQti63Ren/PVVeAgK9oNZb1vH0dM2WNbwByFyhK1Kxc5jkwshyKnJYX/xfrblb2Nz7mZK9CUoKPT2680D/R7g0ohL5dW4JHUyMtCtVqgptI1oWJ4FFdm2x7IMW4DXVf61rrO3rVlhwh0QMgBCB4B3tF2nfBNCUFZXxqHSQxwoOcD+kv0cKDlAuaEcAHetOyNCRnBJ2CWMCB2Bj5N95xqVJKn1dN5AFwIMVaArsX1V59u+qvLqnxfYnlfmgMVwyoaKbUo3n2jofS34dwP/eNujq7/dmhTWmmrJ1+WTVZlFZlUmWZVZZFXZvioNlfUlV4j1imVM+Bh6+fWij38funh1wUHVeb/NkiT9pUm/6YqiTAI+BNTA10KIN89YrtQvnwLUArcLIfa2cFltqvJsNx/1FVBXcfajrhRq60Pcajp7e7UW3IPAPQSC+0D8ZFtrE+9o26NXODg4tkrRz2Symqg0VFJeV06FoeLkY6m+lMLaQgpqCyjUFVKoK6TaVH3atv7O/kR5RjEhcgJRHlF09+1OD98euGpc26TskiS1P+cNdEVR1MCnwGVADrBLUZTFQohDp6w2GYir/xoCfF7/2PKO74Bfbv/rtUoDzl7g5GV79AyDkL7g4me7onb1sz13D7SFuIvPWVfZQgjMwoxVWLFYLZiNVbZHqxmT1YTZaj7tucFiwGg1YrT89WWwGDBYDOjNeurMdegtevQmPXWWOnQmHTqTjhpjDTWm+i9jDbXm2kY/pp+zH4EugUS4RzAoaBCBLoEEuQYR6RFJlEcUblq31ji7kiR1YE25Qh8MpAkhMgAURZkHTAdODfTpwPdCCAFsVxTFS1GUYCFEfksXeIuLC2/3GoZQVAhFQaBgxYoQAoFAiHKsplKsFSmIcoFVWBEILMKCVVhPflmsFqz89bqlaVVanDXOOKodcdW44q5xx1XjSqBrIG4aN9y0brhr3PF28rZ9OXrj5eR18lGj0rR4mSRJ6tyaEuihwPFTXudw9tV3Q+uEAqcFuqIo9wD3AERERDS3rAC4ugYS69cDlaJCQUFRFBSU016rFNXJ16c+qlVqFBTUihqVSoVasb3WqDQnlzsoDiefa1QaNCoNDiqHk48OKge0ai1alRatWouj2hGNWoNWpcXJwQkXBxcc1Y6oVeoL+nySJEkXqimB3tBdQHEB6yCEmA3MBkhISDhreVP0C+hHv4B+F7KpJElSp9aU9nY5QPgpr8OAvAtYR5IkSWpFTQn0XUCcoijRiqJogRuAxWessxi4VbEZClS2Rv25JEmS1LjzVrkIIcyKojwIrMTWbPFbIcRBRVHuq1/+BbAMW5PFNGzNFu9ovSJLkiRJDWlSO3QhxDJsoX3qe1+c8lwA/2jZokmSJEnNYb8+65IkSVKLkoEuSZLUSchAlyRJ6iRkoEuSJHUSiu1+ph0OrCjFQPYFbu4HlLRgcVpKey0XtN+yyXI1jyxX83TGckUKIfwbWmC3QL8YiqLsFkIk2LscZ2qv5YL2WzZZruaR5Wqev1u5ZJWLJElSJyEDXZIkqZPoqIE+294FaER7LRe037LJcjWPLFfz/K3K1SHr0CVJkqSzddQrdEmSJOkMMtAlSZI6iXYb6IqiXKsoykFFUayKoiScsexpRVHSFEVJVRRlYiPb+yiKslpRlKP1j96tUMafFUVJrP/KUhQlsZH1shRF2V+/3u6WLkcDx3tRUZTcU8o2pZH1JtWfwzRFUZ5qg3L9R1GUFEVRkhVF+U1RFK9G1muT83W+z18/HPRH9cuTFUUZ0FplOeWY4YqirFMU5XD9z//DDawzRlGUylO+v/9u7XKdcuxzfm/sdM7iTzkXiYqiVCmK8sgZ67TJOVMU5VtFUYoURTlwyntNyqIW+X0UQrTLL6A7EA+sBxJOeb8HkAQ4AtFAOqBuYPu3gafqnz8FvNXK5X0X+Hcjy7IAvzY8dy8Cj51nHXX9uYsBtPXntEcrl2sC4FD//K3Gvidtcb6a8vmxDQm9HNuMXEOBHW3wvQsGBtQ/dweONFCuMcAfbfXz1JzvjT3OWQPf1wJsnW/a/JwBlwADgAOnvHfeLGqp38d2e4UuhDgshEhtYNF0YJ4QwiCEyMQ2BvvgRtb7rv75d8CVrVJQbFclwHXAT611jFZwcvJvIYQRODH5d6sRQqwSQpjrX27HNrOVvTTl85+c/FwIsR3wUhQluDULJYTIF0LsrX9eDRzGNj9vR9Hm5+wM44B0IcSF9kK/KEKIjUDZGW83JYta5Pex3Qb6OTQ2IfWZAkX9rEn1jwGtWKZRQKEQ4mgjywWwSlGUPfUTZbeFB+v/5f22kX/xmnoeW8ssbFdyDWmL89WUz2/Xc6QoShTQH9jRwOJhiqIkKYqyXFGUnm1VJs7/vbH3z9UNNH5hZa9z1pQsapHz1qQJLlqLoih/AkENLHpWCLGosc0aeK/V2l42sYw3cu6r8xFCiDxFUQKA1YqipNT/JW+VcgGfA69gOy+vYKsOmnXmLhrY9qLPY1POl6IozwJmYG4ju2nx89VQURt474ImP28NiqK4AQuBR4QQVWcs3outSqGm/v7I70BcW5SL839v7HnOtMA04OkGFtvznDVFi5w3uwa6EGL8BWzW1AmpCxVFCRZC5Nf/y1fUGmVUFMUBuAoYeI595NU/FimK8hu2f68uKqCaeu4URfkK+KOBRa0ysXcTztdtwOXAOFFfedjAPlr8fDWg3U5+riiKBluYzxVC/Hrm8lMDXgixTFGUzxRF8RNCtPogVE343thzwvjJwF4hROGZC+x5zmhaFrXIeeuIVS6LgRsURXFUFCUa21/ZnY2sd1v989uAxq74L9Z4IEUIkdPQQkVRXBVFcT/xHNuNwQMNrdtSzqiznNHI8Zoy+XdLl2sS8CQwTQhR28g6bXW+2uXk5/X3Y74BDgsh3mtknaD69VAUZTC23+PS1ixX/bGa8r2x54Txjf6nbK9zVq8pWdQyv4+tfdf3Qr+wBVEOYAAKgZWnLHsW2x3hVGDyKe9/TX2LGMAXWAMcrX/0aaVy/g+474z3QoBl9c9jsN2xTgIOYqt6aO1z9wOwH0iu/6EIPrNc9a+nYGtFkd5G5UrDVk+YWP/1hT3PV0OfH7jvxPcT27/Bn9Yv388pra1asUwjsf2rnXzKeZpyRrkerD83SdhuLg9v7XKd63tj73NWf1wXbAHtecp7bX7OsP1ByQdM9fl1Z2NZ1Bq/j7LrvyRJUifREatcJEmSpAbIQJckSeokZKBLkiR1EjLQJUmSOgkZ6JIkSZ2EDHRJkqROQga6JElSJ/H/5jsFhDtmU3cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for _ in range(6):\n",
    "    i = random.choice(range(len(sub_x)))\n",
    "    linear_output = np.concatenate((random_linear(sub_x[:i]), random_linear(sub_x[i:])))\n",
    "    i2 = random.choice(range(len(sub_x)))\n",
    "    output = np.concatenate((sigmoid(linear_output[:i2]), sigmoid(linear_output[i2:])))\n",
    "    plt.plot(sub_x, output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "effective-waters",
   "metadata": {},
   "source": [
    "\n",
    "## 动态显示函数的变化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "regulation-extreme",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.animation import FuncAnimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "demonstrated-closure",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw(index):\n",
    "    i = random.choice(range(len(sub_x)))\n",
    "    linear_output = np.concatenate((random_linear(sub_x[:i]), random_linear(sub_x[i:])))\n",
    "    i2 = random.choice(range(len(sub_x)))\n",
    "    output = np.concatenate((sigmoid(linear_output[:i2]), sigmoid(linear_output[i2:])))\n",
    "    fig.clear()\n",
    "    plt.plot(sub_x, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "large-flush",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "convinced-simulation",
   "metadata": {},
   "source": [
    "### 先线性变换再sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "nuclear-target",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "/* Put everything inside the global mpl namespace */\n",
       "/* global mpl */\n",
       "window.mpl = {};\n",
       "\n",
       "mpl.get_websocket_type = function () {\n",
       "    if (typeof WebSocket !== 'undefined') {\n",
       "        return WebSocket;\n",
       "    } else if (typeof MozWebSocket !== 'undefined') {\n",
       "        return MozWebSocket;\n",
       "    } else {\n",
       "        alert(\n",
       "            'Your browser does not have WebSocket support. ' +\n",
       "                'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
       "                'Firefox 4 and 5 are also supported but you ' +\n",
       "                'have to enable WebSockets in about:config.'\n",
       "        );\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n",
       "    this.id = figure_id;\n",
       "\n",
       "    this.ws = websocket;\n",
       "\n",
       "    this.supports_binary = this.ws.binaryType !== undefined;\n",
       "\n",
       "    if (!this.supports_binary) {\n",
       "        var warnings = document.getElementById('mpl-warnings');\n",
       "        if (warnings) {\n",
       "            warnings.style.display = 'block';\n",
       "            warnings.textContent =\n",
       "                'This browser does not support binary websocket messages. ' +\n",
       "                'Performance may be slow.';\n",
       "        }\n",
       "    }\n",
       "\n",
       "    this.imageObj = new Image();\n",
       "\n",
       "    this.context = undefined;\n",
       "    this.message = undefined;\n",
       "    this.canvas = undefined;\n",
       "    this.rubberband_canvas = undefined;\n",
       "    this.rubberband_context = undefined;\n",
       "    this.format_dropdown = undefined;\n",
       "\n",
       "    this.image_mode = 'full';\n",
       "\n",
       "    this.root = document.createElement('div');\n",
       "    this.root.setAttribute('style', 'display: inline-block');\n",
       "    this._root_extra_style(this.root);\n",
       "\n",
       "    parent_element.appendChild(this.root);\n",
       "\n",
       "    this._init_header(this);\n",
       "    this._init_canvas(this);\n",
       "    this._init_toolbar(this);\n",
       "\n",
       "    var fig = this;\n",
       "\n",
       "    this.waiting = false;\n",
       "\n",
       "    this.ws.onopen = function () {\n",
       "        fig.send_message('supports_binary', { value: fig.supports_binary });\n",
       "        fig.send_message('send_image_mode', {});\n",
       "        if (fig.ratio !== 1) {\n",
       "            fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n",
       "        }\n",
       "        fig.send_message('refresh', {});\n",
       "    };\n",
       "\n",
       "    this.imageObj.onload = function () {\n",
       "        if (fig.image_mode === 'full') {\n",
       "            // Full images could contain transparency (where diff images\n",
       "            // almost always do), so we need to clear the canvas so that\n",
       "            // there is no ghosting.\n",
       "            fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
       "        }\n",
       "        fig.context.drawImage(fig.imageObj, 0, 0);\n",
       "    };\n",
       "\n",
       "    this.imageObj.onunload = function () {\n",
       "        fig.ws.close();\n",
       "    };\n",
       "\n",
       "    this.ws.onmessage = this._make_on_message_function(this);\n",
       "\n",
       "    this.ondownload = ondownload;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_header = function () {\n",
       "    var titlebar = document.createElement('div');\n",
       "    titlebar.classList =\n",
       "        'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n",
       "    var titletext = document.createElement('div');\n",
       "    titletext.classList = 'ui-dialog-title';\n",
       "    titletext.setAttribute(\n",
       "        'style',\n",
       "        'width: 100%; text-align: center; padding: 3px;'\n",
       "    );\n",
       "    titlebar.appendChild(titletext);\n",
       "    this.root.appendChild(titlebar);\n",
       "    this.header = titletext;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._init_canvas = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var canvas_div = (this.canvas_div = document.createElement('div'));\n",
       "    canvas_div.setAttribute(\n",
       "        'style',\n",
       "        'border: 1px solid #ddd;' +\n",
       "            'box-sizing: content-box;' +\n",
       "            'clear: both;' +\n",
       "            'min-height: 1px;' +\n",
       "            'min-width: 1px;' +\n",
       "            'outline: 0;' +\n",
       "            'overflow: hidden;' +\n",
       "            'position: relative;' +\n",
       "            'resize: both;'\n",
       "    );\n",
       "\n",
       "    function on_keyboard_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.key_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    canvas_div.addEventListener(\n",
       "        'keydown',\n",
       "        on_keyboard_event_closure('key_press')\n",
       "    );\n",
       "    canvas_div.addEventListener(\n",
       "        'keyup',\n",
       "        on_keyboard_event_closure('key_release')\n",
       "    );\n",
       "\n",
       "    this._canvas_extra_style(canvas_div);\n",
       "    this.root.appendChild(canvas_div);\n",
       "\n",
       "    var canvas = (this.canvas = document.createElement('canvas'));\n",
       "    canvas.classList.add('mpl-canvas');\n",
       "    canvas.setAttribute('style', 'box-sizing: content-box;');\n",
       "\n",
       "    this.context = canvas.getContext('2d');\n",
       "\n",
       "    var backingStore =\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        this.context.webkitBackingStorePixelRatio ||\n",
       "        this.context.mozBackingStorePixelRatio ||\n",
       "        this.context.msBackingStorePixelRatio ||\n",
       "        this.context.oBackingStorePixelRatio ||\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        1;\n",
       "\n",
       "    this.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
       "    if (this.ratio !== 1) {\n",
       "        fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n",
       "    }\n",
       "\n",
       "    var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n",
       "        'canvas'\n",
       "    ));\n",
       "    rubberband_canvas.setAttribute(\n",
       "        'style',\n",
       "        'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n",
       "    );\n",
       "\n",
       "    var resizeObserver = new ResizeObserver(function (entries) {\n",
       "        var nentries = entries.length;\n",
       "        for (var i = 0; i < nentries; i++) {\n",
       "            var entry = entries[i];\n",
       "            var width, height;\n",
       "            if (entry.contentBoxSize) {\n",
       "                if (entry.contentBoxSize instanceof Array) {\n",
       "                    // Chrome 84 implements new version of spec.\n",
       "                    width = entry.contentBoxSize[0].inlineSize;\n",
       "                    height = entry.contentBoxSize[0].blockSize;\n",
       "                } else {\n",
       "                    // Firefox implements old version of spec.\n",
       "                    width = entry.contentBoxSize.inlineSize;\n",
       "                    height = entry.contentBoxSize.blockSize;\n",
       "                }\n",
       "            } else {\n",
       "                // Chrome <84 implements even older version of spec.\n",
       "                width = entry.contentRect.width;\n",
       "                height = entry.contentRect.height;\n",
       "            }\n",
       "\n",
       "            // Keep the size of the canvas and rubber band canvas in sync with\n",
       "            // the canvas container.\n",
       "            if (entry.devicePixelContentBoxSize) {\n",
       "                // Chrome 84 implements new version of spec.\n",
       "                canvas.setAttribute(\n",
       "                    'width',\n",
       "                    entry.devicePixelContentBoxSize[0].inlineSize\n",
       "                );\n",
       "                canvas.setAttribute(\n",
       "                    'height',\n",
       "                    entry.devicePixelContentBoxSize[0].blockSize\n",
       "                );\n",
       "            } else {\n",
       "                canvas.setAttribute('width', width * fig.ratio);\n",
       "                canvas.setAttribute('height', height * fig.ratio);\n",
       "            }\n",
       "            canvas.setAttribute(\n",
       "                'style',\n",
       "                'width: ' + width + 'px; height: ' + height + 'px;'\n",
       "            );\n",
       "\n",
       "            rubberband_canvas.setAttribute('width', width);\n",
       "            rubberband_canvas.setAttribute('height', height);\n",
       "\n",
       "            // And update the size in Python. We ignore the initial 0/0 size\n",
       "            // that occurs as the element is placed into the DOM, which should\n",
       "            // otherwise not happen due to the minimum size styling.\n",
       "            if (width != 0 && height != 0) {\n",
       "                fig.request_resize(width, height);\n",
       "            }\n",
       "        }\n",
       "    });\n",
       "    resizeObserver.observe(canvas_div);\n",
       "\n",
       "    function on_mouse_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.mouse_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousedown',\n",
       "        on_mouse_event_closure('button_press')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseup',\n",
       "        on_mouse_event_closure('button_release')\n",
       "    );\n",
       "    // Throttle sequential mouse events to 1 every 20ms.\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousemove',\n",
       "        on_mouse_event_closure('motion_notify')\n",
       "    );\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseenter',\n",
       "        on_mouse_event_closure('figure_enter')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseleave',\n",
       "        on_mouse_event_closure('figure_leave')\n",
       "    );\n",
       "\n",
       "    canvas_div.addEventListener('wheel', function (event) {\n",
       "        if (event.deltaY < 0) {\n",
       "            event.step = 1;\n",
       "        } else {\n",
       "            event.step = -1;\n",
       "        }\n",
       "        on_mouse_event_closure('scroll')(event);\n",
       "    });\n",
       "\n",
       "    canvas_div.appendChild(canvas);\n",
       "    canvas_div.appendChild(rubberband_canvas);\n",
       "\n",
       "    this.rubberband_context = rubberband_canvas.getContext('2d');\n",
       "    this.rubberband_context.strokeStyle = '#000000';\n",
       "\n",
       "    this._resize_canvas = function (width, height, forward) {\n",
       "        if (forward) {\n",
       "            canvas_div.style.width = width + 'px';\n",
       "            canvas_div.style.height = height + 'px';\n",
       "        }\n",
       "    };\n",
       "\n",
       "    // Disable right mouse context menu.\n",
       "    this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n",
       "        event.preventDefault();\n",
       "        return false;\n",
       "    });\n",
       "\n",
       "    function set_focus() {\n",
       "        canvas.focus();\n",
       "        canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    window.setTimeout(set_focus, 100);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'mpl-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'mpl-button-group';\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'mpl-button-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        var button = (fig.buttons[name] = document.createElement('button'));\n",
       "        button.classList = 'mpl-widget';\n",
       "        button.setAttribute('role', 'button');\n",
       "        button.setAttribute('aria-disabled', 'false');\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "\n",
       "        var icon_img = document.createElement('img');\n",
       "        icon_img.src = '_images/' + image + '.png';\n",
       "        icon_img.srcset = '_images/' + image + '_large.png 2x';\n",
       "        icon_img.alt = tooltip;\n",
       "        button.appendChild(icon_img);\n",
       "\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    var fmt_picker = document.createElement('select');\n",
       "    fmt_picker.classList = 'mpl-widget';\n",
       "    toolbar.appendChild(fmt_picker);\n",
       "    this.format_dropdown = fmt_picker;\n",
       "\n",
       "    for (var ind in mpl.extensions) {\n",
       "        var fmt = mpl.extensions[ind];\n",
       "        var option = document.createElement('option');\n",
       "        option.selected = fmt === mpl.default_extension;\n",
       "        option.innerHTML = fmt;\n",
       "        fmt_picker.appendChild(option);\n",
       "    }\n",
       "\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n",
       "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
       "    // which will in turn request a refresh of the image.\n",
       "    this.send_message('resize', { width: x_pixels, height: y_pixels });\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_message = function (type, properties) {\n",
       "    properties['type'] = type;\n",
       "    properties['figure_id'] = this.id;\n",
       "    this.ws.send(JSON.stringify(properties));\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_draw_message = function () {\n",
       "    if (!this.waiting) {\n",
       "        this.waiting = true;\n",
       "        this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    var format_dropdown = fig.format_dropdown;\n",
       "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
       "    fig.ondownload(fig, format);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_resize = function (fig, msg) {\n",
       "    var size = msg['size'];\n",
       "    if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n",
       "        fig._resize_canvas(size[0], size[1], msg['forward']);\n",
       "        fig.send_message('refresh', {});\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n",
       "    var x0 = msg['x0'] / fig.ratio;\n",
       "    var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n",
       "    var x1 = msg['x1'] / fig.ratio;\n",
       "    var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n",
       "    x0 = Math.floor(x0) + 0.5;\n",
       "    y0 = Math.floor(y0) + 0.5;\n",
       "    x1 = Math.floor(x1) + 0.5;\n",
       "    y1 = Math.floor(y1) + 0.5;\n",
       "    var min_x = Math.min(x0, x1);\n",
       "    var min_y = Math.min(y0, y1);\n",
       "    var width = Math.abs(x1 - x0);\n",
       "    var height = Math.abs(y1 - y0);\n",
       "\n",
       "    fig.rubberband_context.clearRect(\n",
       "        0,\n",
       "        0,\n",
       "        fig.canvas.width / fig.ratio,\n",
       "        fig.canvas.height / fig.ratio\n",
       "    );\n",
       "\n",
       "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n",
       "    // Updates the figure title.\n",
       "    fig.header.textContent = msg['label'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n",
       "    var cursor = msg['cursor'];\n",
       "    switch (cursor) {\n",
       "        case 0:\n",
       "            cursor = 'pointer';\n",
       "            break;\n",
       "        case 1:\n",
       "            cursor = 'default';\n",
       "            break;\n",
       "        case 2:\n",
       "            cursor = 'crosshair';\n",
       "            break;\n",
       "        case 3:\n",
       "            cursor = 'move';\n",
       "            break;\n",
       "    }\n",
       "    fig.rubberband_canvas.style.cursor = cursor;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_message = function (fig, msg) {\n",
       "    fig.message.textContent = msg['message'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n",
       "    // Request the server to send over a new figure.\n",
       "    fig.send_draw_message();\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n",
       "    fig.image_mode = msg['mode'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n",
       "    for (var key in msg) {\n",
       "        if (!(key in fig.buttons)) {\n",
       "            continue;\n",
       "        }\n",
       "        fig.buttons[key].disabled = !msg[key];\n",
       "        fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n",
       "    if (msg['mode'] === 'PAN') {\n",
       "        fig.buttons['Pan'].classList.add('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    } else if (msg['mode'] === 'ZOOM') {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.add('active');\n",
       "    } else {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Called whenever the canvas gets updated.\n",
       "    this.send_message('ack', {});\n",
       "};\n",
       "\n",
       "// A function to construct a web socket function for onmessage handling.\n",
       "// Called in the figure constructor.\n",
       "mpl.figure.prototype._make_on_message_function = function (fig) {\n",
       "    return function socket_on_message(evt) {\n",
       "        if (evt.data instanceof Blob) {\n",
       "            /* FIXME: We get \"Resource interpreted as Image but\n",
       "             * transferred with MIME type text/plain:\" errors on\n",
       "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
       "             * to be part of the websocket stream */\n",
       "            evt.data.type = 'image/png';\n",
       "\n",
       "            /* Free the memory for the previous frames */\n",
       "            if (fig.imageObj.src) {\n",
       "                (window.URL || window.webkitURL).revokeObjectURL(\n",
       "                    fig.imageObj.src\n",
       "                );\n",
       "            }\n",
       "\n",
       "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
       "                evt.data\n",
       "            );\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        } else if (\n",
       "            typeof evt.data === 'string' &&\n",
       "            evt.data.slice(0, 21) === 'data:image/png;base64'\n",
       "        ) {\n",
       "            fig.imageObj.src = evt.data;\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        var msg = JSON.parse(evt.data);\n",
       "        var msg_type = msg['type'];\n",
       "\n",
       "        // Call the  \"handle_{type}\" callback, which takes\n",
       "        // the figure and JSON message as its only arguments.\n",
       "        try {\n",
       "            var callback = fig['handle_' + msg_type];\n",
       "        } catch (e) {\n",
       "            console.log(\n",
       "                \"No handler for the '\" + msg_type + \"' message type: \",\n",
       "                msg\n",
       "            );\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        if (callback) {\n",
       "            try {\n",
       "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
       "                callback(fig, msg);\n",
       "            } catch (e) {\n",
       "                console.log(\n",
       "                    \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n",
       "                    e,\n",
       "                    e.stack,\n",
       "                    msg\n",
       "                );\n",
       "            }\n",
       "        }\n",
       "    };\n",
       "};\n",
       "\n",
       "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
       "mpl.findpos = function (e) {\n",
       "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
       "    var targ;\n",
       "    if (!e) {\n",
       "        e = window.event;\n",
       "    }\n",
       "    if (e.target) {\n",
       "        targ = e.target;\n",
       "    } else if (e.srcElement) {\n",
       "        targ = e.srcElement;\n",
       "    }\n",
       "    if (targ.nodeType === 3) {\n",
       "        // defeat Safari bug\n",
       "        targ = targ.parentNode;\n",
       "    }\n",
       "\n",
       "    // pageX,Y are the mouse positions relative to the document\n",
       "    var boundingRect = targ.getBoundingClientRect();\n",
       "    var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n",
       "    var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n",
       "\n",
       "    return { x: x, y: y };\n",
       "};\n",
       "\n",
       "/*\n",
       " * return a copy of an object with only non-object keys\n",
       " * we need this to avoid circular references\n",
       " * http://stackoverflow.com/a/24161582/3208463\n",
       " */\n",
       "function simpleKeys(original) {\n",
       "    return Object.keys(original).reduce(function (obj, key) {\n",
       "        if (typeof original[key] !== 'object') {\n",
       "            obj[key] = original[key];\n",
       "        }\n",
       "        return obj;\n",
       "    }, {});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.mouse_event = function (event, name) {\n",
       "    var canvas_pos = mpl.findpos(event);\n",
       "\n",
       "    if (name === 'button_press') {\n",
       "        this.canvas.focus();\n",
       "        this.canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    var x = canvas_pos.x * this.ratio;\n",
       "    var y = canvas_pos.y * this.ratio;\n",
       "\n",
       "    this.send_message(name, {\n",
       "        x: x,\n",
       "        y: y,\n",
       "        button: event.button,\n",
       "        step: event.step,\n",
       "        guiEvent: simpleKeys(event),\n",
       "    });\n",
       "\n",
       "    /* This prevents the web browser from automatically changing to\n",
       "     * the text insertion cursor when the button is pressed.  We want\n",
       "     * to control all of the cursor setting manually through the\n",
       "     * 'cursor' event from matplotlib */\n",
       "    event.preventDefault();\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n",
       "    // Handle any extra behaviour associated with a key event\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.key_event = function (event, name) {\n",
       "    // Prevent repeat events\n",
       "    if (name === 'key_press') {\n",
       "        if (event.which === this._key) {\n",
       "            return;\n",
       "        } else {\n",
       "            this._key = event.which;\n",
       "        }\n",
       "    }\n",
       "    if (name === 'key_release') {\n",
       "        this._key = null;\n",
       "    }\n",
       "\n",
       "    var value = '';\n",
       "    if (event.ctrlKey && event.which !== 17) {\n",
       "        value += 'ctrl+';\n",
       "    }\n",
       "    if (event.altKey && event.which !== 18) {\n",
       "        value += 'alt+';\n",
       "    }\n",
       "    if (event.shiftKey && event.which !== 16) {\n",
       "        value += 'shift+';\n",
       "    }\n",
       "\n",
       "    value += 'k';\n",
       "    value += event.which.toString();\n",
       "\n",
       "    this._key_event_extra(event, name);\n",
       "\n",
       "    this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n",
       "    if (name === 'download') {\n",
       "        this.handle_save(this, null);\n",
       "    } else {\n",
       "        this.send_message('toolbar_button', { name: name });\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n",
       "    this.message.textContent = tooltip;\n",
       "};\n",
       "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
       "\n",
       "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
       "\n",
       "mpl.default_extension = \"png\";/* global mpl */\n",
       "\n",
       "var comm_websocket_adapter = function (comm) {\n",
       "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
       "    // object with the appropriate methods. Currently this is a non binary\n",
       "    // socket, so there is still some room for performance tuning.\n",
       "    var ws = {};\n",
       "\n",
       "    ws.close = function () {\n",
       "        comm.close();\n",
       "    };\n",
       "    ws.send = function (m) {\n",
       "        //console.log('sending', m);\n",
       "        comm.send(m);\n",
       "    };\n",
       "    // Register the callback with on_msg.\n",
       "    comm.on_msg(function (msg) {\n",
       "        //console.log('receiving', msg['content']['data'], msg);\n",
       "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
       "        ws.onmessage(msg['content']['data']);\n",
       "    });\n",
       "    return ws;\n",
       "};\n",
       "\n",
       "mpl.mpl_figure_comm = function (comm, msg) {\n",
       "    // This is the function which gets called when the mpl process\n",
       "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
       "\n",
       "    var id = msg.content.data.id;\n",
       "    // Get hold of the div created by the display call when the Comm\n",
       "    // socket was opened in Python.\n",
       "    var element = document.getElementById(id);\n",
       "    var ws_proxy = comm_websocket_adapter(comm);\n",
       "\n",
       "    function ondownload(figure, _format) {\n",
       "        window.open(figure.canvas.toDataURL());\n",
       "    }\n",
       "\n",
       "    var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n",
       "\n",
       "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
       "    // web socket which is closed, not our websocket->open comm proxy.\n",
       "    ws_proxy.onopen();\n",
       "\n",
       "    fig.parent_element = element;\n",
       "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
       "    if (!fig.cell_info) {\n",
       "        console.error('Failed to find cell for figure', id, fig);\n",
       "        return;\n",
       "    }\n",
       "    fig.cell_info[0].output_area.element.one(\n",
       "        'cleared',\n",
       "        { fig: fig },\n",
       "        fig._remove_fig_handler\n",
       "    );\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_close = function (fig, msg) {\n",
       "    var width = fig.canvas.width / fig.ratio;\n",
       "    fig.cell_info[0].output_area.element.off(\n",
       "        'cleared',\n",
       "        fig._remove_fig_handler\n",
       "    );\n",
       "\n",
       "    // Update the output cell to use the data from the current canvas.\n",
       "    fig.push_to_output();\n",
       "    var dataURL = fig.canvas.toDataURL();\n",
       "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
       "    // the notebook keyboard shortcuts fail.\n",
       "    IPython.keyboard_manager.enable();\n",
       "    fig.parent_element.innerHTML =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "    fig.close_ws(fig, msg);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.close_ws = function (fig, msg) {\n",
       "    fig.send_message('closing', msg);\n",
       "    // fig.ws.close()\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n",
       "    // Turn the data on the canvas into data in the output cell.\n",
       "    var width = this.canvas.width / this.ratio;\n",
       "    var dataURL = this.canvas.toDataURL();\n",
       "    this.cell_info[1]['text/html'] =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Tell IPython that the notebook contents must change.\n",
       "    IPython.notebook.set_dirty(true);\n",
       "    this.send_message('ack', {});\n",
       "    var fig = this;\n",
       "    // Wait a second, then push the new image to the DOM so\n",
       "    // that it is saved nicely (might be nice to debounce this).\n",
       "    setTimeout(function () {\n",
       "        fig.push_to_output();\n",
       "    }, 1000);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'btn-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'btn-group';\n",
       "    var button;\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'btn-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        button = fig.buttons[name] = document.createElement('button');\n",
       "        button.classList = 'btn btn-default';\n",
       "        button.href = '#';\n",
       "        button.title = name;\n",
       "        button.innerHTML = '<i class=\"fa ' + image + ' fa-lg\"></i>';\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    // Add the status bar.\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message pull-right';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "\n",
       "    // Add the close button to the window.\n",
       "    var buttongrp = document.createElement('div');\n",
       "    buttongrp.classList = 'btn-group inline pull-right';\n",
       "    button = document.createElement('button');\n",
       "    button.classList = 'btn btn-mini btn-primary';\n",
       "    button.href = '#';\n",
       "    button.title = 'Stop Interaction';\n",
       "    button.innerHTML = '<i class=\"fa fa-power-off icon-remove icon-large\"></i>';\n",
       "    button.addEventListener('click', function (_evt) {\n",
       "        fig.handle_close(fig, {});\n",
       "    });\n",
       "    button.addEventListener(\n",
       "        'mouseover',\n",
       "        on_mouseover_closure('Stop Interaction')\n",
       "    );\n",
       "    buttongrp.appendChild(button);\n",
       "    var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n",
       "    titlebar.insertBefore(buttongrp, titlebar.firstChild);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._remove_fig_handler = function (event) {\n",
       "    var fig = event.data.fig;\n",
       "    fig.close_ws(fig, {});\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (el) {\n",
       "    el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (el) {\n",
       "    // this is important to make the div 'focusable\n",
       "    el.setAttribute('tabindex', 0);\n",
       "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
       "    // off when our div gets focus\n",
       "\n",
       "    // location in version 3\n",
       "    if (IPython.notebook.keyboard_manager) {\n",
       "        IPython.notebook.keyboard_manager.register_events(el);\n",
       "    } else {\n",
       "        // location in version 2\n",
       "        IPython.keyboard_manager.register_events(el);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (event, _name) {\n",
       "    var manager = IPython.notebook.keyboard_manager;\n",
       "    if (!manager) {\n",
       "        manager = IPython.keyboard_manager;\n",
       "    }\n",
       "\n",
       "    // Check for shift+enter\n",
       "    if (event.shiftKey && event.which === 13) {\n",
       "        this.canvas_div.blur();\n",
       "        // select the cell after this one\n",
       "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
       "        IPython.notebook.select(index + 1);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    fig.ondownload(fig, null);\n",
       "};\n",
       "\n",
       "mpl.find_output_cell = function (html_output) {\n",
       "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
       "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
       "    // IPython event is triggered only after the cells have been serialised, which for\n",
       "    // our purposes (turning an active figure into a static one), is too late.\n",
       "    var cells = IPython.notebook.get_cells();\n",
       "    var ncells = cells.length;\n",
       "    for (var i = 0; i < ncells; i++) {\n",
       "        var cell = cells[i];\n",
       "        if (cell.cell_type === 'code') {\n",
       "            for (var j = 0; j < cell.output_area.outputs.length; j++) {\n",
       "                var data = cell.output_area.outputs[j];\n",
       "                if (data.data) {\n",
       "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
       "                    data = data.data;\n",
       "                }\n",
       "                if (data['text/html'] === html_output) {\n",
       "                    return [cell, data, j];\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "};\n",
       "\n",
       "// Register the function which deals with the matplotlib target/channel.\n",
       "// The kernel may be null if the page has been refreshed.\n",
       "if (IPython.notebook.kernel !== null) {\n",
       "    IPython.notebook.kernel.comm_manager.register_target(\n",
       "        'matplotlib',\n",
       "        mpl.mpl_figure_comm\n",
       "    );\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"data:,\" width=\"0\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.animation.FuncAnimation at 0x28a19cd0dc0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fig = plt.gcf()\n",
    "FuncAnimation(fig, draw, interval=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unknown-speaking",
   "metadata": {},
   "source": [
    "### 先线性变换再sigmoid再线性变换，这里的线性变化是打乱了前后顺序的实现了非线性输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "outstanding-cooper",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 增加反向(负)变换\n",
    "def random_linear2(x):\n",
    "    k, b = random.normalvariate(0, 1), random.normalvariate(0, 1)\n",
    "    return k * x + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "changing-adaptation",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw2(index):\n",
    "    i = random.choice(range(len(sub_x)))\n",
    "    linear_output = np.concatenate((random_linear2(sub_x[:i]), random_linear2(sub_x[i:])))\n",
    "    i2 = random.choice(range(len(sub_x)))\n",
    "    output = np.concatenate((random_linear2(sigmoid(linear_output[:i2])), random_linear2(sigmoid(linear_output[i2:]))))\n",
    "    fig2.clear()\n",
    "    plt.plot(sub_x, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "promotional-taiwan",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.animation.FuncAnimation at 0x28a19ce0e50>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fig2 = plt.gcf()\n",
    "FuncAnimation(fig2, draw2, interval=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "christian-transfer",
   "metadata": {},
   "source": [
    "基于上面的动态展示我们可以发现由三次变换我们就可以生成各种各样的复杂函数，在实际过程中，神经网络正式由这种特性可以拟合各种复杂特征分布。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "olive-sending",
   "metadata": {},
   "source": [
    "基于这种特性我们可以将寻找分布的工作让神经网络承担这部分工作，我们要思考的是如何求未知函数的导数？"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "logical-tuner",
   "metadata": {},
   "source": [
    "### 我们可以使用基本模块(函数)组合拼接就可以产生非常复杂的函数\n",
    "上面的操作为：线性函数 + 非线性函数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wanted-cartridge",
   "metadata": {},
   "source": [
    "## 问题"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "persistent-rendering",
   "metadata": {},
   "source": [
    "### 神经网络\n",
    "之前用的网络都不超过三层，超过三层的就叫**深度神经网络**，为什么之前都不超过三层呢？\n",
    "\n",
    "原因有如下两点：\n",
    "* 数据量不足：神经网络要获得好的性能需要使用大量数据，要拟合的参数每增加一个，理论上就需要将数据集增强一个数量级，但是上世纪没有那么多数据；\n",
    "* 算力不足，之前计算机技术比较差，大规模运算并没有普及。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "heavy-performer",
   "metadata": {},
   "source": [
    "#### 关于上面三次变换后的函数，我们如何求导呢？明显手动求导是非常复杂的，这里科学家提出了链式法则进行求导。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "specialized-position",
   "metadata": {},
   "source": [
    "## 参数间关系的图谱建立"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAC/CAYAAAAl+nzaAAAZLklEQVR4Ae2dv24jORKHG3AmYDNnlyvd7B5h9xHW0As43ODgeAEnF90Gkw0udXiB32ATv8BkBja6+BxMcr4/c4AOJYsWrWZL7BZZLJJfA4ZaFLvJ+n5V1aWWRA8DGwQg0BOBzfX19Zerq6tvwzBse/sTu8X+YRg2PYmOrRCAAAQgAAGTBFar1af1ev318fFx+/r6uu1xE7vFfuEgPEwKxaQgAAEIQAACnRDYyAW516LkuBATDsKDOyideD9mQgACEICAPQLyUYbcMWA7EBAe+4947AnGjCAAAQhAAAKtE5DvWnDX5FCYyJ7w2H/3pnX5sQ8CEIAABCBgksDHKzPPdgT2Xwg2KRiTggAEIAABCLROgHIkQIDipHW3xz4IQAACELBMIHBppmlfnPwyDEPuP8u+wdwgAAEIQAACRQhQiQQIKBUnsp4MGwQgAAEIQAACRwQCl2aalD7WoTg5ckaeQmAOAVaOZOXIOf5C37oIUIkECFCc1OXEzLYzAqwc+fazQlaO7Mzx+zI3cGnO2/T09LS9vb09uRrt/f399uHhIe9ETpyd4qSvIMDaugiwcqSXvFg5si7nZbbRBDwv19k9V5xIYSLFAcVJtIZ0hEA/BFg5cpyoWTmyH//vyNKxo2du8YsTKfrlLooUJP4+d0468kBMhcAcAqwcOc7QrBw5x4PoWwmBsaNnbnHFycvLy64wkefHG8VJJd7DNCFQgMBxvuD5duv+lXwBORgSAlkIqMe1K07u7u4mP7qhOMmiNSeFQBME1JNWDQMqfVGuCQfCiCoIqIedFCcSR58/f95KgSJ3JI83ipMqfIdJQqAIgeN8wfPDnZPcq0bOOX8R52DQZgiox7W7cyJFyVQRMtWuNVmlNyGsc9JMGGGIJgGtPFDVOPukNad4yNmX5KYZEW2OpR5/fnHy/Py8vbm52cr3T/yN4qRNZ8MqCKQg4OcK9vcElN5RxepHcRJLin5TBNRj2y9OZHD5ybAUI/5GcTIlF+0QgICfK9jfE6A4ITAaI0BsBwgoxTlvLhoLJszRIRAI2bxNx++oQqN18o4qVmGSWywp+k0RCIVZ920UJ1PuQjsEyhNQT1DnihMpTCRpdLByZKz6FCexpOg3RUA9zmsYkOJkyl1oh0B5Auo5xC9O/NUi/X3unHxwDIqTDzh4soCAepzXMCDFyQJP4hAIKBFQzyGuOGHlyGiFKU6iUdFxgoB6nNcwIMXJhLfQDAEDBNRziCtOWDkyWn2Kk2hUdJwgoB7nNQxIcTLhLTRDwAAB9RwixYkkBVaOjFaf4iQaFR1DBPgfWuM0Jx8jC5cQr8RtxG9ioJyuDwLjqM3c4u6csHJktIOR3KJR0TFEgP8+Pk5qiv99nPgNOSVtEDhDYBy1mVv84qTzlSPPSPP+MsntHQU7Cwls1uv1V3lDwLbd/Z8f4TEMw2YhzzmHEb9zaNEXAnsC6rnKL05k8I5Xjox1QpJbLCn6TRJYrVaf5IIsdwx6LVLEbrFfOAiPSVhpXyB+0/LkbJ0QUC9OahhQ6YtysS5GcoslRb9zBDbyEc/+uxbiV139id1iv9IdE6cF8etI8AiBGQRqqBXU50hxMsOD6AoBCJwiQHFyig6vQWCCgPqFv4YBKU4mvIVmCEBgLgGKk7nE6A8BuQizjQlQnBAbEIBAIgIUJ4lAcpq+CIyvzLRIMrGUUCzNpa/owFoIXE6A+L2cIWfokAClSIAAxUmHkYDJEMhDgOIkD1fO2jIBVo4cVybyc0OllSNjXYvkFkuKfhCwR4D4tacJM7JOgJUjx8WJ4sqRse5BcoslRT8I2CNA/NrThBlVQICVI736RO6aKK4cGeseJLdYUvSDgD0CxK89TZhRDQRYOfJtOesCK0fGugfJLZYU/SBgjwDxa08TZlQRAVaO1F85MtY9SG6xpOgHAXsEiF97mjAjCEAgAQGSWwKInAIChQgQv4XAMywEIJCXAMktL1/ODoGcBIjfnHQ5NwQgUIwAya0YegaGwMUEiN+LEXICCEDAIgGSm0VVmBME4ggQv3Gc6AUBCFRGgORWmWBMFwIeAeLXg8EuBCDQDgGSWztaYkl/BIjf/jTHYgh0QYDk1oXMGNkoAeK3UWExCwK9EyC59e4B2F8zAeK3ZvWYOwQgMEmA5DaJhhcgYJ4A8WteIiYIAQgsIUByW0KNYyBggwDxa0MHZgEBCCQmQHJLDJTTQUCRAPGrCJuhIAABPQIkNz3WjASB1ASI39REOR8EIGCCAMnNhAxMAgKLCBC/i7BxEAQgYJ0Ayc26QswPAtMEiN9pNrwCAQhUTIDkVrF4TL17AsRv9y4AAAi0SYDk1qauWNUHAeK3D52xEgLdESC5dSc5BjdEgPhtSExMgQAEDgRIbgcW7EGgNgLEb22KMV8IQCCKAMktChOdIGCSAPFrUhYmBQEIXEqA5HYpQY6HQDkCxG859owMAQhkJEByywiXU0MgMwHiNzNgTg8BCJQhQHIrw51RIZCCAPGbgiLngAAEzBEguZmThAlBIJoA8RuNio4QgEBNBEhuNanFXCHwkQDx+5EHzyAAgUYIkNwaERIzuiRA/HYpO0ZDoH0CJLf2NcbCdgkQv+1qi2UQ6JoAya1r+TG+cgLEb+UCMn0IQCBMgOQW5kIrBGogQPzWoBJzhAAEZhMguc1GxgEQMEOA+DUjBROBAARSEiC5paTJuSCgS4D41eXNaBCAgBIBkpsSaIaBQAYCxG8GqJwSAhAoT4DkVl4DZgCBpQSI36XkOA4CEDBNgORmWh4mB4GTBIjfk3h4EQIQqJUAya1W5Zg3BIaB+MULIACBJgmQ3JqUFaM6IUD8diI0ZkKgNwIkt94Ux96WCBC/LamJLRCAwDsBkts7CnYgUB0B4rc6yZgwBCAQQ4DkFkOJPhCwSYD4takLs4IABC4kQHK7ECCHQ6AgAeK3IHyGrp/A5vr6+svV1dW34e3b5RJQ3fyJ3WL/MAwbg1JaTG74i11/OeXC6Kavm2b8oq++vqfiLfY1dAvptlqtPq3X66+Pj4/b19fXbY+b2C32CwfhEetRSv00k9tZk/CX7S5ODPtLUEN0K6abSvyibzF9g/EW24hu07pt5ILca1FyXIgJB+Fh7A6KSnKLDCb8xXMao/4SkhLdyummEb/oW07fULzFtqHblG7yUYa8A2Q7EBAe+494Yh0sdz+N5OZs+M7thB7xl4OfuD2D/jKSDt2cWodHRd1SxC9xeZAuak9R31G8xTYQl2Mp33WT71pw1+QjIOGx/+5NrI/l7pciucXM8Q/DMPw+DMOfh2EIJkP85aOvyDOD/jLSGt2K6nZp/BKXY/nOthCXZxGZ7ODrZnKCpSe1/zLwKMkXarg0uc2Z9m/DMPxvGIZ/TxQppaUxOb4xfwnpbZJb6Ukp6ZYifonLBc6ipG8o3mLbFljV/iFOt/YtXWChgxPrYZn7pUhusVP84zAM/9zb/59AkbKAZvuHGPOXkNbti7DAQiXdUsQvcWlX31C8xbYtsKr9Q1xctm/pAgsdnFgPy9xPktsvin9/39sv48qfFCny91d5zjYmYMxfQu44njQtzr9DvFK2pYpf4nKmzxKXM4EZ6e50MzIdW9NwcFJmqAvOpVmYyFgkwZnuaMxfQq4206I+uivplip+icuZbqmkbyjeYttmWtRHd6dbH9bOtNLBifWwhvpx+3imr0j3CvxlgVXtH1KBbi61EJcL3LECfRdY1f4hTrf2LV1goYPjMkNHj3zxrk1/WWBV+4dUFOfE5QJ3rEDfBVa1f4jTTdVS+ZnQ7e3t9unp6eS4Dw8P2/v7+5N9cr7o4HRUlIip8pPFfwzD8JepnxILF+1N/OCcL+Avw59OaCbaZpft+fl5e3Nzs315eck+VqoBKonz6uJS4nXPdvvjjz8W8wkD+haPy1OxIjH7ww8/bOXR0uZ0U51TTHEiFxqZ3LkLUs6JOzidFSdi7r+GYfjvxM+IVS5yx7qeK07wl52XmtPtWEeLzyuKc3P6TsWlvPH0c/dUPw1/MKCvOd187hQnHg1XnHz+/Hn7/fff74oQubi4TRxZ7qzI676Du9e1Hg04dam66Od9gRL6GbG54gR/eXcTVd2E+z5G3uP0+M6JKxolzu/u7nb9puLfnc9/ly13YOT58Tgpc0BFca6qbwxj0Uz+zm1SrPi6nuuf8nUD+prTzedLceLRcMnJOesUHElsMY7vnTrprgGnfr/qFNh5cReE/U+I/QXZknKOOVlMEsRfdl6ioptcbOQNhMSy/EnhIXHsFyd+XLsiQ3R08e+Ol3OJr4l+7rXjffER/3wxPhPbp7I4V9E3ll1MXMq5RE+nd+y5U/Uzoq8p3Xy2ueLKH2PJvtNtybGLj/ETkDuJOLk4sL/Jc2kvte3hvL9r4/nuo56/CQftLSYJ4i+79WhC/iof0SXVzS9OfF/wi5NjPdzz4/j3j5FzhXKBtEuBI99nkf4ptwbiOrm+sXxj4rL0xc+wvsV08/UtrY8/F39/r5vuxcYlJ0lwbgslJJfMXB/tRwenwF0LC0OaqvRjkiD+onfnRGJReLvEL/uy+YXGsWZOn+P494+Rc/i5wN1VcePIx0PSP+VWWZxXFZeilWjm5/qU2sWcy4i+pnTzuYlGfCF2T8QlJ5fQjp87cC6Zuefaj0acukShIp+RvgY+znFz0ZZid8GSi9apDX8Ziujm4lcuQH6hcayHe+73Fz39Y+S5K07cR0Huwpb5zonzbcuPRfQ9FXPHBajfV3TLUUz6Y8TsG8jj5nTzuVGceDRccnKfQU7BccnMO1R114BTl0qU5r5dfioJOqfAX/R+ZeWzlnhe8p0TV3TEFicyZo6LXUVxXk1cTuV0F6uajwb0Naebz9+SVv68nG5+W/Z9V5z4v9Zxicof3E+AfrvWvoNTqkIoNK7Z9RT2erx/lHB8JwV/0VvnxMWw00TYy3ZcaEi79JEvv//666+7uyLuWBfzx8e4OydyPne8nEPyhbyhccelygN7GwqFW/SwVcWlr5vzEfcDiFS6xZ7HgL7m1zmRot/pJI853gTE6uX6Od3ccx49Ag5OdPpopyMrUXp+ELtbgb/EmpK8nytIXBGTfIALTliBbi6zEJcLdK5A3wVWtX+I0619SxdY6OC4zNDRI//Do01/WWDV8kPkDsc+hnaPx3e6lp857ZEVxTlxuUD6CvRdYFX7hzjd2rd0gYUOjpGiJNV/NY09D//9dKbPGPOXkNvOtKiP7kq6xcbduX7E5Uy3VNI3FG+xbTMt6qO7060Pa2da6eDEeljmfvLu81ziSvk6SbBufwm540yL+uiuFOep4pe4nOmWSvqG4i22baZFfXR3uvVh7UwrHZxYD8vcT5Kb1sbt45m+It2N+UvIVxZY1f4hSrqliF/icoE7KukbirfYtgVWtX+I0619SxdY6ODEeljmfimSW+wU+eJd/f4S0nqBVe0fohTnKeKXuFzgjkr6huIttm2BVe0fstPt6urqm3ybnu1AQHgIl1jvUuiXIrnFTFN+svj7if9IPOAvBz9xewb9ZaQ1ujm1Do+Kul0av8TlQbboPUV9R/EW20BcjuV81+36+vrL4+PjuEfHLcJDuMQ6mEK/S5PbnCl+d6oz/jIODIP+MpIQ3YrqliJ+icuxhCdbiMuTeMy+6Ou2Wa/XX6VaYdvu/jOq8BiGYTPK8OUaUiS3VLPHX7xAkbgx6C8hrdGtnG4a8Yu+5fQNxVtsG7qd0m21Wn2SBCsVS69Fitgt9gsH4RHrWUr9NJJbtCn4y1sRa9hfglqiWzHdVOIXfYvpG4y32EZ0O6/bRm797r9rIcHU1Z/Yvf8ox9IdE+ffKsnNDRb5iL+8ffRn0V9OSYhu+rppxi/66ut7Kt5iX0O3OnWL1bfZfprJrVmIGAaBQgSI30LgGRYCEMhLgOSWly9nh0BOAsRvTrqcGwIQKEaA5FYMPQND4GICxO/FCDkBBCBgkQDJzaIqzAkCcQSI3zhO9IIABCojQHKrTDCmCwGPAPHrwWAXAhBohwDJrR0tsaQ/AsRvf5pjMQS6IEBy60JmjGyUAPHbqLCYBYHeCZDcevcA7K+ZAPFbs3rMHQIQmCRAcptEwwsQME+A+DUvEROEAASWECC5LaHGMRCwQYD4taEDs4AABBITILklBsrpIKBIgPhVhM1QEICAHgGSmx5rRoJAagLEb2qinA8CEDBBgORmQgYmAYFFBIjfRdg4CAIQsE6A5GZdIeYHgWkCxO80G16BAAQqJkByq1g8pt49AeK3excAAATaJEBya1NXrOqDAPHbh85YCYHuCJDcupMcgxsiQPw2JCamQAACBwIktwML9iBQGwHitzbFmC8EIBBFgOQWhYlOEDBJgPg1KQuTggAELiVAcruUIMdDoBwB4rcce0aGAAQyEiC5ZYTLqSGQmQDxmxkwp4cABMoQILmV4c6oEEhBgPhNQZFzQAAC5giQ3MxJwoQgEE2A+I1GRUcIQKAmAiS3mtRirhD4SID4/ciDZxCAQCMESG6NCIkZXRIgfruUHaMh0D4Bklv7GmNhuwSI33a1xTIIdE2A5Na1/BhfOQHit3IBmT4EIBAmQHILc6EVAjUQIH5rUIk5QgACswmQ3GYj4wAImCFA/JqRgolAAAIpCZDcUtLkXBDQJUD86vJmNAhAQIkAyU0JNMNAIAMB4jcDVE4JAQiUJ0ByK68BM4DAUgLE71JyHAcBCJgmQHIzLQ+Tg8BJAsTvSTy8CAEI1EqA5FarcswbAsNA/OIFEIBAkwRIbk3KilGdECB+OxEaMyHQGwGSW2+KY29LBIjfltTEFghA4J0Aye0dBTsQqI4A8VudZEwYAhCIIUByi6FEHwjYJED82tSFWUEAAhcSILldCJDDIVCQAPFbED5DQwAC+QiQ3PKx7e3Mm+vr6y9XV1ffhrdfkYhvdfMndov9wzBsFIUnfhVhMxQEIKBHgOSmx7rZkVar1af1ev318fFx+/r6uu1xE7vFfuEgPJTEJn6VQDMMBCCgS4Dkpsu7xdE2ckHutSg5LsSEg/BQuoNC/LYYUdgEAQiwiBM+cBkB+ShD7hiwHQgIj/1HPJfBPX80xcl5RvSAAAQqJEByq1A0S1OW71pw1+RQmMie8Nh/9ya3VMRvbsKcHwIQKEKA5FYEe1ODfrwy82xHYP9l4NxCE7+5CXN+CECgCAGSWxHsTQ1KORIgQHHSlI9jDAQgoEyA4kQZeIPDBS7NNFGcNOjpmAQBCKgRoDhRQ93sQFQiAQIUJ836O4ZBAAIKBChOFCA3PkTg0kwTxUnjXo95TRBg5Uj9lSNjHYfiJJYU/aYIUIkECFCcTLkL7RAwQICVI99+Vlhg5chY9SlOYknRb4pA4NJME8XJlLvQDoHyBFg50svRyitHxqpPcRJLin5TBDwvZ9cRoDiZchfaIVCYACtHujR1eFRcOTJWfYqTWFL0myJwcPACe/f399uHh4fdyPIG4Pb2dvv09FRgJh+HpDiZchfaIVCYACtHfkxW8kxx5chY9SlOYknRb4rA2NEVWySm7u7uts/Pz7uixBUqilMIDkVxMuUutEOgPIFg0PbeqJS0YtWnOIklRb8pAsVDWgqTn376aXtzc7N9eXkpPh+ZwD7Ocz/+NiUK7RCAwDQBE0nC2iQoTqYdhleqJGAixPyPdyxMyFicV+lYTBoCuQhYyBHm5mAsaXHnJJf393Pe4jEm3zGRj3bcxzvFJ3S4c9KPF2ApBCoiYCFHmJsDxUlFHsxUYwgUjTH5GEe+BCuP8vGOFCjyPZTSm7E4j9GRPhDohkDp/GByfGNJizsn3YRjNkOLxVno1znyhVj5iKf0ZizOs4nPiSFQI4HS+cHk+MaSFsVJjZFla84m46z0pIzFuS2PYTYQKEygdH4wOb6xpEVxUjhIGhjeZJyVnpSxOG/AzTABAukIlM4PJsc3lrQoTtL5e69nMhlnpSdlLM579U3shkCQQNH84P+0MPTZdKnJGUtaFCdB16VxBoFSoWR6XGNxPkNOukKgfQJFk0fnK0fGehfFSSwp+k0RKBrnVgenOJlyF9ohUJ5A8bzR8cqRUnTE/LHCZPk4qX0GxePc4gQoTmp3a+bfMgETOcP/eMfChEhaLbt8f7bxP7TGWUXu2gqX/rwBiyFQB4Fx1Cq3sHJkHY7CLOslwH8fHyc1g/99vF4HY+YQyEBgHLWKLawcmUFRTgmBMYHNer3+KncL2N7+87jwGIZhM0ZFCwQgYIFAsVwV+nUOK0dacAnm0CKB1Wr1SS7Icseg1yJF7Bb7hYPwaFFnbIJAKwSKFSeWB+Y7J624N3YcEdjIRzz771rEfBm7qT5it9jPHZMjr+ApBAwSsFwjFJsbxYlBT2VKEIAABCDQDYFiBYDlgSlOuvF/DIUABCAAAYMELNcIxeZGcWLQU5kSBCAAAQh0Q6BYAWB5YIqTbvwfQyEAAQhAwCAByzVCsblRnBj0VKYEAQhAAAJ9EGDlyHH9Iz83ZOXIPvwfKyEAAQhAwCABVo4cFyesHGnQUZkSBCAAAQh0RYCVI736RO6asHJkV/6PsRCAAAQgYJEAK0e+LWfNypEWvZM5QQACEIBAzwRYOZKVI3v2f2yHAAQgAAEjBP4Pls77I0DxJj8AAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "id": "amended-court",
   "metadata": {},
   "source": [
    "#### 我们人可以知道链式法则，那么计算机如何知道对某个参数的链式求导路径？\n",
    "这种参数间的关系，我们如何让计算机知道？如下图的关系，我们会使用图来保存这种关系，并方便计算机计算。\n",
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "crude-setup",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "opponent-calvin",
   "metadata": {},
   "outputs": [],
   "source": [
    "computing_graph = {\n",
    "    \"k1\": [\"L1\"],\n",
    "    \"b1\": [\"L1\"],\n",
    "    \"x\": [\"L1\"],\n",
    "    \"L1\": [\"sigmoid\"],\n",
    "    \"k2\": [\"L2\"],\n",
    "    \"sigmoid\": [\"L2\"],\n",
    "    \"b2\": [\"L2\"],\n",
    "    \"L2\": [\"Loss\"],\n",
    "    \"y\": [\"Loss\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "encouraging-spyware",
   "metadata": {},
   "outputs": [],
   "source": [
    "nx.draw(nx.DiGraph(computing_graph), with_labels=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "intimate-numbers",
   "metadata": {},
   "source": [
    "很明显这是一个有向图，反向时，只需要将所有箭头逆置就可以了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "handled-happening",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_output(graph, node):\n",
    "    outputs = []\n",
    "    for n, links in graph.items():\n",
    "        if node == n: outputs.append(links[0])\n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "human-silence",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['L1']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_output(computing_graph, \"k1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acceptable-sequence",
   "metadata": {},
   "source": [
    "### 如何获得k1的偏导？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "light-venue",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入打印LaTeX的库\n",
    "from IPython.display import Latex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "muslim-thesis",
   "metadata": {},
   "outputs": [],
   "source": [
    "def partial_chain(target):\n",
    "    computing_order = []\n",
    "    # 获取node\n",
    "    out = get_output(computing_graph, target)[0]\n",
    "    computing_order.append(target)\n",
    "\n",
    "    while out:\n",
    "        computing_order.append(out)\n",
    "        out = get_output(computing_graph, out)\n",
    "        if out: out = out[0]\n",
    "\n",
    "    order = []\n",
    "\n",
    "    for index, n in enumerate(computing_order[:-1]):\n",
    "        order.append((computing_order[index+1], n))\n",
    "\n",
    "    partial_str = r\"\\frac{{\\partial {}}}{{\\partial {}}}\"\n",
    "\n",
    "    # Latex(\"$\" + r\"\\times \".join([partial_str.format(a, b) for a, b in order[::-1]]) + \"$\")\n",
    "    return Latex(\"$\" + r\"\\times \".join([partial_str.format(a, b) for a, b in order[::-1]]) + \"$\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "nonprofit-notice",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\frac{\\partial Loss}{\\partial L2}\\times \\frac{\\partial L2}{\\partial b2}$"
      ],
      "text/plain": [
       "<IPython.core.display.Latex object>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_chain(\"b2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "inappropriate-sodium",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\frac{\\partial Loss}{\\partial L2}\\times \\frac{\\partial L2}{\\partial k2}$"
      ],
      "text/plain": [
       "<IPython.core.display.Latex object>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_chain(\"k2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "breeding-douglas",
   "metadata": {},
   "source": [
    "### 此时，我们不难发现所有的参数可以都可以计算其偏导了\n",
    "但是这中间会有多少工作是重复的呢？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "defined-recording",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\frac{\\partial Loss}{\\partial L2}\\times \\frac{\\partial L2}{\\partial sigmoid}\\times \\frac{\\partial sigmoid}{\\partial L1}\\times \\frac{\\partial L1}{\\partial k1}$"
      ],
      "text/plain": [
       "<IPython.core.display.Latex object>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_chain(\"k1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "historical-statistics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\frac{\\partial Loss}{\\partial L2}\\times \\frac{\\partial L2}{\\partial sigmoid}\\times \\frac{\\partial sigmoid}{\\partial L1}\\times \\frac{\\partial L1}{\\partial b1}$"
      ],
      "text/plain": [
       "<IPython.core.display.Latex object>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_chain(\"b1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "assisted-happiness",
   "metadata": {},
   "source": [
    "### 上面的参数求偏导，我们发现其重复是很高的，如果我们的参数很多，那么这种额外计算的开销就会很大\n",
    "### 为了加速在模型训练阶段的计算开销，我们可以先将偏导计算的结果存起来，等到反向传播时，再将对应的节点偏导数取出做计算就可以了。\n",
    "\n",
    "如我们按下面的顺序记录偏导，对每个参数我们只需要对应的元素取出相乘即可\n",
    "\n",
    "$$\n",
    "\\begin{equation}\n",
    "    \\begin{aligned}\n",
    "         &\\frac{\\partial {Loss}}{\\partial {L2}},\\\\\n",
    "         &\\frac{\\partial {Loss}}{\\partial {y}},\\\\\n",
    "         &\\frac{\\partial {L2}}{\\partial {sigmoid}},\\\\\n",
    "         &\\frac{\\partial {sigmoid}}{\\partial {L1}},\\\\\n",
    "         &\\frac{\\partial {L1}}{\\partial {k1}},\\\\\n",
    "         &\\frac{\\partial {L1}}{\\partial {b1}},\\\\\n",
    "         &\\frac{\\partial {L2}}{\\partial {k2}},\\\\\n",
    "         &\\frac{\\partial {L2}}{\\partial {b2}},\n",
    "    \\end{aligned}\n",
    "\\end{equation}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "parliamentary-shipping",
   "metadata": {},
   "source": [
    "关于拓扑排序的顺序怎么求呢？有一个很好的顺序我们就可以实现所有参数的偏导了！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "quality-response",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "parental-benchmark",
   "metadata": {},
   "source": [
    "## lesson 3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cognitive-employment",
   "metadata": {},
   "source": [
    "### 实现拓扑排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "southwest-association",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import reduce\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "basic-likelihood",
   "metadata": {},
   "outputs": [],
   "source": [
    "def topologic(graph:dict):\n",
    "    \"\"\"\n",
    "    \n",
    "    \"\"\"\n",
    "    sorted_node = []\n",
    "    while graph:\n",
    "        all_nodes_have_inputs = reduce(lambda a, b: a + b, list(graph.values()))\n",
    "        all_nodes_have_outputs = list(graph.keys())\n",
    "        \n",
    "        all_node_only_have_outputs_no_inputs = set(all_nodes_have_outputs) - set(all_nodes_have_inputs)\n",
    "        \n",
    "        if all_node_only_have_outputs_no_inputs:\n",
    "            node = random.choice(list(all_node_only_have_outputs_no_inputs))\n",
    "            sorted_node.append(node)\n",
    "            if len(graph) == 1: sorted_node += graph[node]\n",
    "            graph.pop(node)\n",
    "            \n",
    "            \n",
    "            for _, links in graph.items():\n",
    "                if node in links: links.remove(node)\n",
    "        else:\n",
    "            raise TypeError(\"This graph has circle, which cannot get topological order\")\n",
    "            \n",
    "    return sorted_node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "funded-genius",
   "metadata": {},
   "outputs": [],
   "source": [
    "computing_graph = {\n",
    "    \"k1\": [\"L1\"],\n",
    "    \"b1\": [\"L1\"],\n",
    "    \"x\": [\"L1\"],\n",
    "    \"L1\": [\"sigmoid\"],\n",
    "    \"k2\": [\"L2\"],\n",
    "    \"sigmoid\": [\"L2\"],\n",
    "    \"b2\": [\"L2\"],\n",
    "    \"L2\": [\"Loss\"],\n",
    "    \"y\": [\"Loss\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "passing-sociology",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['x', 'k2', 'b1', 'y', 'b2', 'k1', 'L1', 'sigmoid', 'L2', 'Loss']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topologic(computing_graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liked-first",
   "metadata": {},
   "source": [
    "## Node类声明"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "diagnostic-worry",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Node(object):\n",
    "    \n",
    "    def __init__(self, inputs=[], name=\"\", is_trainable=False):\n",
    "        self.inputs = inputs\n",
    "        self.outputs = []\n",
    "        self.name = name\n",
    "        self.value = None\n",
    "        self.gradients = dict() # 存储Loss对node的偏导\n",
    "        self.is_trainable = is_trainable\n",
    "        \n",
    "        for node in inputs:\n",
    "            node.outputs.append(self)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"Node: {self.name}\"\n",
    "    \n",
    "    def forward(self):\n",
    "        print(f\"I am {self.name}, I calculate myself value!!\")\n",
    "        \n",
    "    def backward(self):\n",
    "        for node in self.outputs:\n",
    "            print(f\"正在计算：\\t∂{node.name} / ∂{self.name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "aggregate-google",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Placeholder(Node):\n",
    "    \n",
    "    def __init__(self, name=\"\", is_trainable=False):\n",
    "        super(Placeholder, self).__init__(name=name, is_trainable=is_trainable)\n",
    "        \n",
    "    def forward(self):\n",
    "        #print(f\"I am {self.name}, I was assigned value {self.value} by human.\")\n",
    "        pass\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"Placeholder: {self.name}\"\n",
    "    \n",
    "    def backward(self):\n",
    "        # print(\"I am: {}\".format(self.name))\n",
    "        # print(\"I got myself gradients: {}\".format(self.outputs[0].gradients[self]))\n",
    "        self.gradients[self] = self.outputs[0].gradients[self]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "sized-mineral",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(Node):\n",
    "    \n",
    "    def __init__(self, x, k, b, name=\"\"):\n",
    "        super(Linear, self).__init__(inputs=[x, k, b], name=name)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"Linear: {self.name}\"\n",
    "    \n",
    "    def forward(self):\n",
    "        self.value = self.inputs[0].value * self.inputs[1].value + self.inputs[2].value\n",
    "        # print(f\"I am {self.name}, I calculate myself value = {self.value}!!\")\n",
    "        \n",
    "    def backward(self):\n",
    "        # print(\"I am: {}\".format(self.name))\n",
    "        # self.gradients[self.inputs[0]] = \" * \".join([self.outputs[0].gradients[self], f\"∂{self.name}/∂{self.inputs[0].name}\"])\n",
    "        # self.gradients[self.inputs[1]] = \" * \".join([self.outputs[0].gradients[self], f\"∂{self.name}/∂{self.inputs[1].name}\"])\n",
    "        # self.gradients[self.inputs[2]] = \" * \".join([self.outputs[0].gradients[self], f\"∂{self.name}/∂{self.inputs[2].name}\"])\n",
    "        # print(\"self.gradients[self.inputs[0]]: {}\".format(self.gradients[self.inputs[0]]))\n",
    "        # print(\"self.gradients[self.inputs[1]]: {}\".format(self.gradients[self.inputs[1]]))\n",
    "        # print(\"self.gradients[self.inputs[2]]: {}\".format(self.gradients[self.inputs[2]]))\n",
    "        \n",
    "        self.gradients[self.inputs[0]] = self.outputs[0].gradients[self] * self.inputs[1].value\n",
    "        self.gradients[self.inputs[1]] = self.outputs[0].gradients[self] * self.inputs[0].value\n",
    "        self.gradients[self.inputs[2]] = self.outputs[0].gradients[self]\n",
    "        # print(\"self.gradients[self.inputs[0]] | ∂{}: {}\".format(self.inputs[0].name, self.gradients[self.inputs[0]]))\n",
    "        # print(\"self.gradients[self.inputs[1]] | ∂{}: {}\".format(self.inputs[1].name, self.gradients[self.inputs[1]]))\n",
    "        # print(\"self.gradients[self.inputs[2]] | ∂{}: {}\".format(self.inputs[2].name, self.gradients[self.inputs[2]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "valid-library",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sigmoid(Node):\n",
    "    \n",
    "    def __init__(self, x, name=\"\"):\n",
    "        super(Sigmoid, self).__init__(inputs=[x], name=name)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"Sigmoid: {self.name}\"\n",
    "    \n",
    "    @staticmethod\n",
    "    def _sigmoid(x):\n",
    "        return 1 / (1 + np.exp(-x))\n",
    "    \n",
    "    def forward(self):\n",
    "        self.value = self._sigmoid(self.inputs[0].value)\n",
    "        # print(f\"I am {self.name}, I calculate myself value = {self.value}!!\")\n",
    "        \n",
    "    def backward(self):\n",
    "        # print(\"I am: {}\".format(self.name))\n",
    "        # self.gradients[self.inputs[0]] = \" * \".join([self.outputs[0].gradients[self], f\"∂{self.name}/∂{self.inputs[0].name}\"])\n",
    "        # print(\"self.gradients[self.inputs[0]]: {}\".format(self.gradients[self.inputs[0]]))\n",
    "        \n",
    "        # self.inputs[0]即为输入的x\n",
    "        self.gradients[self.inputs[0]] = self.outputs[0].gradients[self] * self._sigmoid(self.inputs[0].value) * (1 - self._sigmoid(self.inputs[0].value))\n",
    "        # print(\"self.gradients[self.inputs[0]] | ∂{}: {}\".format(self.inputs[0].name, self.gradients[self.inputs[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "identical-procurement",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MSE(Node):\n",
    "    \n",
    "    def __init__(self, y, yhat, name=\"\"):\n",
    "        super(MSE, self).__init__(inputs=[y, yhat], name=name)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"Sigmoid: {self.name}\"\n",
    "    \n",
    "    def forward(self):\n",
    "        self.value = np.mean((self.inputs[0].value - self.inputs[1].value)**2)\n",
    "        # print(f\"I am {self.name}, I calculate myself value = {self.value}!!\")\n",
    "        \n",
    "    def backward(self):\n",
    "        # print(\"I am: {}\".format(self.name))\n",
    "        # self.gradients[self.inputs[0]] = f\"∂{self.name}/∂{self.inputs[0].name}\"\n",
    "        # self.gradients[self.inputs[1]] = f\"∂{self.name}/∂{self.inputs[1].name}\"\n",
    "        # print(\"self.gradients[self.inputs[0]]: {}\".format(self.gradients[self.inputs[0]]))\n",
    "        # print(\"self.gradients[self.inputs[1]]: {}\".format(self.gradients[self.inputs[1]]))\n",
    "        \n",
    "        # 对y求导\n",
    "        self.gradients[self.inputs[0]] = 2 * np.mean(self.inputs[0].value - self.inputs[1].value)\n",
    "        # 对yhat求导\n",
    "        self.gradients[self.inputs[1]] = 2 * np.mean(self.inputs[1].value - self.inputs[0].value)\n",
    "        # print(\"self.gradients[self.inputs[0]] | ∂{}: {}\".format(self.inputs[0].name, self.gradients[self.inputs[0]]))\n",
    "        # print(\"self.gradients[self.inputs[1]] | ∂{}: {}\".format(self.inputs[1].name, self.gradients[self.inputs[1]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "excellent-gross",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_x = Placeholder(name=\"x\")\n",
    "node_y = Placeholder(name=\"y\")\n",
    "node_k1 = Placeholder(name=\"k1\", is_trainable=True)\n",
    "node_b1 = Placeholder(name=\"b1\", is_trainable=True)\n",
    "node_k2 = Placeholder(name=\"k2\", is_trainable=True)\n",
    "node_b2 = Placeholder(name=\"b2\", is_trainable=True)\n",
    "node_L1 = Linear(node_x, node_k1, node_b1, \"L1\")\n",
    "node_sigmoid = Sigmoid(node_L1, \"sigmoid\")\n",
    "node_L2 = Linear(node_sigmoid, node_k2, node_b2, \"L2\")\n",
    "node_Loss = MSE(node_L2, node_y, \"Loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "several-commonwealth",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_k1.is_trainable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "lovely-accountability",
   "metadata": {},
   "outputs": [],
   "source": [
    "# need_expand = [node_x, node_y, node_k1, node_k2, node_b1, node_b2]\n",
    "feed_dict = {\n",
    "    node_x: random.random(),\n",
    "    node_y: random.random(),\n",
    "    node_k1: random.random(),\n",
    "    node_k2: random.random(),\n",
    "    node_b1: random.random(),\n",
    "    node_b2: random.random(),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "minimal-remedy",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "seasonal-definition",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_feed_dict_to_graph(feed_dict):\n",
    "    need_expand = [n for n in feed_dict]\n",
    "    # 计算图初始化\n",
    "    computing_graph = defaultdict(list)\n",
    "\n",
    "    while need_expand:\n",
    "        n = need_expand.pop(0)\n",
    "\n",
    "        if n in computing_graph: continue  # 在计算图中就跳过\n",
    "        \n",
    "        # 给需要人工输入的节点初始化值\n",
    "        if isinstance(n, Placeholder): n.value = feed_dict[n]\n",
    "\n",
    "        for m in n.outputs:\n",
    "            # 以need_expand列表第一个元素为key，将outputs的元素加入字典的values\n",
    "            computing_graph[n].append(m)\n",
    "            # 将need_expand列表元素的outputs也加入need_expand列表\n",
    "            need_expand.append(m)\n",
    "    return computing_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "fatty-twelve",
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_dict = convert_feed_dict_to_graph(feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "aggressive-miller",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_nodes = topologic(graph_dict)  # topologic排序之后会将计算图的defaultdict置空，这里应该是使用了深复制"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "encouraging-phase",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Placeholder: b1,\n",
       " Placeholder: k1,\n",
       " Placeholder: b2,\n",
       " Placeholder: y,\n",
       " Placeholder: k2,\n",
       " Placeholder: x,\n",
       " Linear: L1,\n",
       " Sigmoid: sigmoid,\n",
       " Linear: L2,\n",
       " Sigmoid: Loss]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_nodes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "separated-rebecca",
   "metadata": {},
   "source": [
    "## 模拟神经网络计算过程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "compressed-combination",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feedforward\n",
    "def forward(graph_sorted_nodes):\n",
    "    for node in graph_sorted_nodes:\n",
    "        node.forward()\n",
    "        if isinstance(node, MSE):\n",
    "            # print(\"Loss value: {}\".format(node.value))\n",
    "            pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "simplified-salem",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backward\n",
    "def backward(graph_sorted_nodes):\n",
    "    for node in graph_sorted_nodes[::-1]:\n",
    "        node.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "remarkable-forth",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_one_epoch(graph_sorted_nodes):\n",
    "    forward(graph_sorted_nodes)\n",
    "    backward(graph_sorted_nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "lucky-exposure",
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimize\n",
    "def optimize(graph_nodes, learning_rate=1e-3):\n",
    "    for node in graph_nodes:\n",
    "        if node.is_trainable:\n",
    "            node.value = node.value - node.gradients[node] * learning_rate\n",
    "            cmp = \"large\" if node.gradients[node] > 0 else \"small\"\n",
    "            # print(\"{} value is too {}, I need update myself to {}.\".format(node.name, cmp, node.value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "regulated-washer",
   "metadata": {},
   "source": [
    "## 一次完整的前向传播、反向传播、参数更新"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "binding-opposition",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_on_one_epoch(graph_nodes):\n",
    "    run_one_epoch(graph_nodes)\n",
    "    optimize(graph_nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "christian-volleyball",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_on_one_epoch(sorted_nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "german-fitting",
   "metadata": {},
   "source": [
    "### 我们设置epoch进行多次迭代"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "adjustable-auction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.36735948405008284,\n",
       " 0.36544188956843987,\n",
       " 0.36353417795732734,\n",
       " 0.3616362991029785,\n",
       " 0.35974820314117745,\n",
       " 0.3578698404560312,\n",
       " 0.35600116167875084,\n",
       " 0.3541421176864361,\n",
       " 0.35229265960086736,\n",
       " 0.35045273878730326]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_epochs = 10\n",
    "\n",
    "def train(graph_nodes, epochs=100):\n",
    "    loss_history = []\n",
    "    \n",
    "    for _ in range(epochs):\n",
    "        run_one_epoch(graph_nodes)\n",
    "        __loss_node = sorted_nodes[-1]\n",
    "        assert isinstance(__loss_node, MSE)\n",
    "\n",
    "        loss_history.append(__loss_node.value)\n",
    "        optimize(graph_nodes)\n",
    "    return loss_history\n",
    "\n",
    "train(sorted_nodes, epochs=train_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "manual-trail",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_loss = train(sorted_nodes, epochs=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "healthy-shooting",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "functioning-consensus",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "/* Put everything inside the global mpl namespace */\n",
       "/* global mpl */\n",
       "window.mpl = {};\n",
       "\n",
       "mpl.get_websocket_type = function () {\n",
       "    if (typeof WebSocket !== 'undefined') {\n",
       "        return WebSocket;\n",
       "    } else if (typeof MozWebSocket !== 'undefined') {\n",
       "        return MozWebSocket;\n",
       "    } else {\n",
       "        alert(\n",
       "            'Your browser does not have WebSocket support. ' +\n",
       "                'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
       "                'Firefox 4 and 5 are also supported but you ' +\n",
       "                'have to enable WebSockets in about:config.'\n",
       "        );\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n",
       "    this.id = figure_id;\n",
       "\n",
       "    this.ws = websocket;\n",
       "\n",
       "    this.supports_binary = this.ws.binaryType !== undefined;\n",
       "\n",
       "    if (!this.supports_binary) {\n",
       "        var warnings = document.getElementById('mpl-warnings');\n",
       "        if (warnings) {\n",
       "            warnings.style.display = 'block';\n",
       "            warnings.textContent =\n",
       "                'This browser does not support binary websocket messages. ' +\n",
       "                'Performance may be slow.';\n",
       "        }\n",
       "    }\n",
       "\n",
       "    this.imageObj = new Image();\n",
       "\n",
       "    this.context = undefined;\n",
       "    this.message = undefined;\n",
       "    this.canvas = undefined;\n",
       "    this.rubberband_canvas = undefined;\n",
       "    this.rubberband_context = undefined;\n",
       "    this.format_dropdown = undefined;\n",
       "\n",
       "    this.image_mode = 'full';\n",
       "\n",
       "    this.root = document.createElement('div');\n",
       "    this.root.setAttribute('style', 'display: inline-block');\n",
       "    this._root_extra_style(this.root);\n",
       "\n",
       "    parent_element.appendChild(this.root);\n",
       "\n",
       "    this._init_header(this);\n",
       "    this._init_canvas(this);\n",
       "    this._init_toolbar(this);\n",
       "\n",
       "    var fig = this;\n",
       "\n",
       "    this.waiting = false;\n",
       "\n",
       "    this.ws.onopen = function () {\n",
       "        fig.send_message('supports_binary', { value: fig.supports_binary });\n",
       "        fig.send_message('send_image_mode', {});\n",
       "        if (fig.ratio !== 1) {\n",
       "            fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n",
       "        }\n",
       "        fig.send_message('refresh', {});\n",
       "    };\n",
       "\n",
       "    this.imageObj.onload = function () {\n",
       "        if (fig.image_mode === 'full') {\n",
       "            // Full images could contain transparency (where diff images\n",
       "            // almost always do), so we need to clear the canvas so that\n",
       "            // there is no ghosting.\n",
       "            fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
       "        }\n",
       "        fig.context.drawImage(fig.imageObj, 0, 0);\n",
       "    };\n",
       "\n",
       "    this.imageObj.onunload = function () {\n",
       "        fig.ws.close();\n",
       "    };\n",
       "\n",
       "    this.ws.onmessage = this._make_on_message_function(this);\n",
       "\n",
       "    this.ondownload = ondownload;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_header = function () {\n",
       "    var titlebar = document.createElement('div');\n",
       "    titlebar.classList =\n",
       "        'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n",
       "    var titletext = document.createElement('div');\n",
       "    titletext.classList = 'ui-dialog-title';\n",
       "    titletext.setAttribute(\n",
       "        'style',\n",
       "        'width: 100%; text-align: center; padding: 3px;'\n",
       "    );\n",
       "    titlebar.appendChild(titletext);\n",
       "    this.root.appendChild(titlebar);\n",
       "    this.header = titletext;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._init_canvas = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var canvas_div = (this.canvas_div = document.createElement('div'));\n",
       "    canvas_div.setAttribute(\n",
       "        'style',\n",
       "        'border: 1px solid #ddd;' +\n",
       "            'box-sizing: content-box;' +\n",
       "            'clear: both;' +\n",
       "            'min-height: 1px;' +\n",
       "            'min-width: 1px;' +\n",
       "            'outline: 0;' +\n",
       "            'overflow: hidden;' +\n",
       "            'position: relative;' +\n",
       "            'resize: both;'\n",
       "    );\n",
       "\n",
       "    function on_keyboard_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.key_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    canvas_div.addEventListener(\n",
       "        'keydown',\n",
       "        on_keyboard_event_closure('key_press')\n",
       "    );\n",
       "    canvas_div.addEventListener(\n",
       "        'keyup',\n",
       "        on_keyboard_event_closure('key_release')\n",
       "    );\n",
       "\n",
       "    this._canvas_extra_style(canvas_div);\n",
       "    this.root.appendChild(canvas_div);\n",
       "\n",
       "    var canvas = (this.canvas = document.createElement('canvas'));\n",
       "    canvas.classList.add('mpl-canvas');\n",
       "    canvas.setAttribute('style', 'box-sizing: content-box;');\n",
       "\n",
       "    this.context = canvas.getContext('2d');\n",
       "\n",
       "    var backingStore =\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        this.context.webkitBackingStorePixelRatio ||\n",
       "        this.context.mozBackingStorePixelRatio ||\n",
       "        this.context.msBackingStorePixelRatio ||\n",
       "        this.context.oBackingStorePixelRatio ||\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        1;\n",
       "\n",
       "    this.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
       "    if (this.ratio !== 1) {\n",
       "        fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n",
       "    }\n",
       "\n",
       "    var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n",
       "        'canvas'\n",
       "    ));\n",
       "    rubberband_canvas.setAttribute(\n",
       "        'style',\n",
       "        'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n",
       "    );\n",
       "\n",
       "    var resizeObserver = new ResizeObserver(function (entries) {\n",
       "        var nentries = entries.length;\n",
       "        for (var i = 0; i < nentries; i++) {\n",
       "            var entry = entries[i];\n",
       "            var width, height;\n",
       "            if (entry.contentBoxSize) {\n",
       "                if (entry.contentBoxSize instanceof Array) {\n",
       "                    // Chrome 84 implements new version of spec.\n",
       "                    width = entry.contentBoxSize[0].inlineSize;\n",
       "                    height = entry.contentBoxSize[0].blockSize;\n",
       "                } else {\n",
       "                    // Firefox implements old version of spec.\n",
       "                    width = entry.contentBoxSize.inlineSize;\n",
       "                    height = entry.contentBoxSize.blockSize;\n",
       "                }\n",
       "            } else {\n",
       "                // Chrome <84 implements even older version of spec.\n",
       "                width = entry.contentRect.width;\n",
       "                height = entry.contentRect.height;\n",
       "            }\n",
       "\n",
       "            // Keep the size of the canvas and rubber band canvas in sync with\n",
       "            // the canvas container.\n",
       "            if (entry.devicePixelContentBoxSize) {\n",
       "                // Chrome 84 implements new version of spec.\n",
       "                canvas.setAttribute(\n",
       "                    'width',\n",
       "                    entry.devicePixelContentBoxSize[0].inlineSize\n",
       "                );\n",
       "                canvas.setAttribute(\n",
       "                    'height',\n",
       "                    entry.devicePixelContentBoxSize[0].blockSize\n",
       "                );\n",
       "            } else {\n",
       "                canvas.setAttribute('width', width * fig.ratio);\n",
       "                canvas.setAttribute('height', height * fig.ratio);\n",
       "            }\n",
       "            canvas.setAttribute(\n",
       "                'style',\n",
       "                'width: ' + width + 'px; height: ' + height + 'px;'\n",
       "            );\n",
       "\n",
       "            rubberband_canvas.setAttribute('width', width);\n",
       "            rubberband_canvas.setAttribute('height', height);\n",
       "\n",
       "            // And update the size in Python. We ignore the initial 0/0 size\n",
       "            // that occurs as the element is placed into the DOM, which should\n",
       "            // otherwise not happen due to the minimum size styling.\n",
       "            if (width != 0 && height != 0) {\n",
       "                fig.request_resize(width, height);\n",
       "            }\n",
       "        }\n",
       "    });\n",
       "    resizeObserver.observe(canvas_div);\n",
       "\n",
       "    function on_mouse_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.mouse_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousedown',\n",
       "        on_mouse_event_closure('button_press')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseup',\n",
       "        on_mouse_event_closure('button_release')\n",
       "    );\n",
       "    // Throttle sequential mouse events to 1 every 20ms.\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousemove',\n",
       "        on_mouse_event_closure('motion_notify')\n",
       "    );\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseenter',\n",
       "        on_mouse_event_closure('figure_enter')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseleave',\n",
       "        on_mouse_event_closure('figure_leave')\n",
       "    );\n",
       "\n",
       "    canvas_div.addEventListener('wheel', function (event) {\n",
       "        if (event.deltaY < 0) {\n",
       "            event.step = 1;\n",
       "        } else {\n",
       "            event.step = -1;\n",
       "        }\n",
       "        on_mouse_event_closure('scroll')(event);\n",
       "    });\n",
       "\n",
       "    canvas_div.appendChild(canvas);\n",
       "    canvas_div.appendChild(rubberband_canvas);\n",
       "\n",
       "    this.rubberband_context = rubberband_canvas.getContext('2d');\n",
       "    this.rubberband_context.strokeStyle = '#000000';\n",
       "\n",
       "    this._resize_canvas = function (width, height, forward) {\n",
       "        if (forward) {\n",
       "            canvas_div.style.width = width + 'px';\n",
       "            canvas_div.style.height = height + 'px';\n",
       "        }\n",
       "    };\n",
       "\n",
       "    // Disable right mouse context menu.\n",
       "    this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n",
       "        event.preventDefault();\n",
       "        return false;\n",
       "    });\n",
       "\n",
       "    function set_focus() {\n",
       "        canvas.focus();\n",
       "        canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    window.setTimeout(set_focus, 100);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'mpl-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'mpl-button-group';\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'mpl-button-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        var button = (fig.buttons[name] = document.createElement('button'));\n",
       "        button.classList = 'mpl-widget';\n",
       "        button.setAttribute('role', 'button');\n",
       "        button.setAttribute('aria-disabled', 'false');\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "\n",
       "        var icon_img = document.createElement('img');\n",
       "        icon_img.src = '_images/' + image + '.png';\n",
       "        icon_img.srcset = '_images/' + image + '_large.png 2x';\n",
       "        icon_img.alt = tooltip;\n",
       "        button.appendChild(icon_img);\n",
       "\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    var fmt_picker = document.createElement('select');\n",
       "    fmt_picker.classList = 'mpl-widget';\n",
       "    toolbar.appendChild(fmt_picker);\n",
       "    this.format_dropdown = fmt_picker;\n",
       "\n",
       "    for (var ind in mpl.extensions) {\n",
       "        var fmt = mpl.extensions[ind];\n",
       "        var option = document.createElement('option');\n",
       "        option.selected = fmt === mpl.default_extension;\n",
       "        option.innerHTML = fmt;\n",
       "        fmt_picker.appendChild(option);\n",
       "    }\n",
       "\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n",
       "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
       "    // which will in turn request a refresh of the image.\n",
       "    this.send_message('resize', { width: x_pixels, height: y_pixels });\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_message = function (type, properties) {\n",
       "    properties['type'] = type;\n",
       "    properties['figure_id'] = this.id;\n",
       "    this.ws.send(JSON.stringify(properties));\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_draw_message = function () {\n",
       "    if (!this.waiting) {\n",
       "        this.waiting = true;\n",
       "        this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    var format_dropdown = fig.format_dropdown;\n",
       "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
       "    fig.ondownload(fig, format);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_resize = function (fig, msg) {\n",
       "    var size = msg['size'];\n",
       "    if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n",
       "        fig._resize_canvas(size[0], size[1], msg['forward']);\n",
       "        fig.send_message('refresh', {});\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n",
       "    var x0 = msg['x0'] / fig.ratio;\n",
       "    var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n",
       "    var x1 = msg['x1'] / fig.ratio;\n",
       "    var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n",
       "    x0 = Math.floor(x0) + 0.5;\n",
       "    y0 = Math.floor(y0) + 0.5;\n",
       "    x1 = Math.floor(x1) + 0.5;\n",
       "    y1 = Math.floor(y1) + 0.5;\n",
       "    var min_x = Math.min(x0, x1);\n",
       "    var min_y = Math.min(y0, y1);\n",
       "    var width = Math.abs(x1 - x0);\n",
       "    var height = Math.abs(y1 - y0);\n",
       "\n",
       "    fig.rubberband_context.clearRect(\n",
       "        0,\n",
       "        0,\n",
       "        fig.canvas.width / fig.ratio,\n",
       "        fig.canvas.height / fig.ratio\n",
       "    );\n",
       "\n",
       "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n",
       "    // Updates the figure title.\n",
       "    fig.header.textContent = msg['label'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n",
       "    var cursor = msg['cursor'];\n",
       "    switch (cursor) {\n",
       "        case 0:\n",
       "            cursor = 'pointer';\n",
       "            break;\n",
       "        case 1:\n",
       "            cursor = 'default';\n",
       "            break;\n",
       "        case 2:\n",
       "            cursor = 'crosshair';\n",
       "            break;\n",
       "        case 3:\n",
       "            cursor = 'move';\n",
       "            break;\n",
       "    }\n",
       "    fig.rubberband_canvas.style.cursor = cursor;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_message = function (fig, msg) {\n",
       "    fig.message.textContent = msg['message'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n",
       "    // Request the server to send over a new figure.\n",
       "    fig.send_draw_message();\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n",
       "    fig.image_mode = msg['mode'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n",
       "    for (var key in msg) {\n",
       "        if (!(key in fig.buttons)) {\n",
       "            continue;\n",
       "        }\n",
       "        fig.buttons[key].disabled = !msg[key];\n",
       "        fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n",
       "    if (msg['mode'] === 'PAN') {\n",
       "        fig.buttons['Pan'].classList.add('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    } else if (msg['mode'] === 'ZOOM') {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.add('active');\n",
       "    } else {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Called whenever the canvas gets updated.\n",
       "    this.send_message('ack', {});\n",
       "};\n",
       "\n",
       "// A function to construct a web socket function for onmessage handling.\n",
       "// Called in the figure constructor.\n",
       "mpl.figure.prototype._make_on_message_function = function (fig) {\n",
       "    return function socket_on_message(evt) {\n",
       "        if (evt.data instanceof Blob) {\n",
       "            /* FIXME: We get \"Resource interpreted as Image but\n",
       "             * transferred with MIME type text/plain:\" errors on\n",
       "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
       "             * to be part of the websocket stream */\n",
       "            evt.data.type = 'image/png';\n",
       "\n",
       "            /* Free the memory for the previous frames */\n",
       "            if (fig.imageObj.src) {\n",
       "                (window.URL || window.webkitURL).revokeObjectURL(\n",
       "                    fig.imageObj.src\n",
       "                );\n",
       "            }\n",
       "\n",
       "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
       "                evt.data\n",
       "            );\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        } else if (\n",
       "            typeof evt.data === 'string' &&\n",
       "            evt.data.slice(0, 21) === 'data:image/png;base64'\n",
       "        ) {\n",
       "            fig.imageObj.src = evt.data;\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        var msg = JSON.parse(evt.data);\n",
       "        var msg_type = msg['type'];\n",
       "\n",
       "        // Call the  \"handle_{type}\" callback, which takes\n",
       "        // the figure and JSON message as its only arguments.\n",
       "        try {\n",
       "            var callback = fig['handle_' + msg_type];\n",
       "        } catch (e) {\n",
       "            console.log(\n",
       "                \"No handler for the '\" + msg_type + \"' message type: \",\n",
       "                msg\n",
       "            );\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        if (callback) {\n",
       "            try {\n",
       "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
       "                callback(fig, msg);\n",
       "            } catch (e) {\n",
       "                console.log(\n",
       "                    \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n",
       "                    e,\n",
       "                    e.stack,\n",
       "                    msg\n",
       "                );\n",
       "            }\n",
       "        }\n",
       "    };\n",
       "};\n",
       "\n",
       "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
       "mpl.findpos = function (e) {\n",
       "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
       "    var targ;\n",
       "    if (!e) {\n",
       "        e = window.event;\n",
       "    }\n",
       "    if (e.target) {\n",
       "        targ = e.target;\n",
       "    } else if (e.srcElement) {\n",
       "        targ = e.srcElement;\n",
       "    }\n",
       "    if (targ.nodeType === 3) {\n",
       "        // defeat Safari bug\n",
       "        targ = targ.parentNode;\n",
       "    }\n",
       "\n",
       "    // pageX,Y are the mouse positions relative to the document\n",
       "    var boundingRect = targ.getBoundingClientRect();\n",
       "    var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n",
       "    var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n",
       "\n",
       "    return { x: x, y: y };\n",
       "};\n",
       "\n",
       "/*\n",
       " * return a copy of an object with only non-object keys\n",
       " * we need this to avoid circular references\n",
       " * http://stackoverflow.com/a/24161582/3208463\n",
       " */\n",
       "function simpleKeys(original) {\n",
       "    return Object.keys(original).reduce(function (obj, key) {\n",
       "        if (typeof original[key] !== 'object') {\n",
       "            obj[key] = original[key];\n",
       "        }\n",
       "        return obj;\n",
       "    }, {});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.mouse_event = function (event, name) {\n",
       "    var canvas_pos = mpl.findpos(event);\n",
       "\n",
       "    if (name === 'button_press') {\n",
       "        this.canvas.focus();\n",
       "        this.canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    var x = canvas_pos.x * this.ratio;\n",
       "    var y = canvas_pos.y * this.ratio;\n",
       "\n",
       "    this.send_message(name, {\n",
       "        x: x,\n",
       "        y: y,\n",
       "        button: event.button,\n",
       "        step: event.step,\n",
       "        guiEvent: simpleKeys(event),\n",
       "    });\n",
       "\n",
       "    /* This prevents the web browser from automatically changing to\n",
       "     * the text insertion cursor when the button is pressed.  We want\n",
       "     * to control all of the cursor setting manually through the\n",
       "     * 'cursor' event from matplotlib */\n",
       "    event.preventDefault();\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n",
       "    // Handle any extra behaviour associated with a key event\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.key_event = function (event, name) {\n",
       "    // Prevent repeat events\n",
       "    if (name === 'key_press') {\n",
       "        if (event.which === this._key) {\n",
       "            return;\n",
       "        } else {\n",
       "            this._key = event.which;\n",
       "        }\n",
       "    }\n",
       "    if (name === 'key_release') {\n",
       "        this._key = null;\n",
       "    }\n",
       "\n",
       "    var value = '';\n",
       "    if (event.ctrlKey && event.which !== 17) {\n",
       "        value += 'ctrl+';\n",
       "    }\n",
       "    if (event.altKey && event.which !== 18) {\n",
       "        value += 'alt+';\n",
       "    }\n",
       "    if (event.shiftKey && event.which !== 16) {\n",
       "        value += 'shift+';\n",
       "    }\n",
       "\n",
       "    value += 'k';\n",
       "    value += event.which.toString();\n",
       "\n",
       "    this._key_event_extra(event, name);\n",
       "\n",
       "    this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n",
       "    if (name === 'download') {\n",
       "        this.handle_save(this, null);\n",
       "    } else {\n",
       "        this.send_message('toolbar_button', { name: name });\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n",
       "    this.message.textContent = tooltip;\n",
       "};\n",
       "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
       "\n",
       "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
       "\n",
       "mpl.default_extension = \"png\";/* global mpl */\n",
       "\n",
       "var comm_websocket_adapter = function (comm) {\n",
       "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
       "    // object with the appropriate methods. Currently this is a non binary\n",
       "    // socket, so there is still some room for performance tuning.\n",
       "    var ws = {};\n",
       "\n",
       "    ws.close = function () {\n",
       "        comm.close();\n",
       "    };\n",
       "    ws.send = function (m) {\n",
       "        //console.log('sending', m);\n",
       "        comm.send(m);\n",
       "    };\n",
       "    // Register the callback with on_msg.\n",
       "    comm.on_msg(function (msg) {\n",
       "        //console.log('receiving', msg['content']['data'], msg);\n",
       "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
       "        ws.onmessage(msg['content']['data']);\n",
       "    });\n",
       "    return ws;\n",
       "};\n",
       "\n",
       "mpl.mpl_figure_comm = function (comm, msg) {\n",
       "    // This is the function which gets called when the mpl process\n",
       "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
       "\n",
       "    var id = msg.content.data.id;\n",
       "    // Get hold of the div created by the display call when the Comm\n",
       "    // socket was opened in Python.\n",
       "    var element = document.getElementById(id);\n",
       "    var ws_proxy = comm_websocket_adapter(comm);\n",
       "\n",
       "    function ondownload(figure, _format) {\n",
       "        window.open(figure.canvas.toDataURL());\n",
       "    }\n",
       "\n",
       "    var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n",
       "\n",
       "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
       "    // web socket which is closed, not our websocket->open comm proxy.\n",
       "    ws_proxy.onopen();\n",
       "\n",
       "    fig.parent_element = element;\n",
       "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
       "    if (!fig.cell_info) {\n",
       "        console.error('Failed to find cell for figure', id, fig);\n",
       "        return;\n",
       "    }\n",
       "    fig.cell_info[0].output_area.element.one(\n",
       "        'cleared',\n",
       "        { fig: fig },\n",
       "        fig._remove_fig_handler\n",
       "    );\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_close = function (fig, msg) {\n",
       "    var width = fig.canvas.width / fig.ratio;\n",
       "    fig.cell_info[0].output_area.element.off(\n",
       "        'cleared',\n",
       "        fig._remove_fig_handler\n",
       "    );\n",
       "\n",
       "    // Update the output cell to use the data from the current canvas.\n",
       "    fig.push_to_output();\n",
       "    var dataURL = fig.canvas.toDataURL();\n",
       "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
       "    // the notebook keyboard shortcuts fail.\n",
       "    IPython.keyboard_manager.enable();\n",
       "    fig.parent_element.innerHTML =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "    fig.close_ws(fig, msg);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.close_ws = function (fig, msg) {\n",
       "    fig.send_message('closing', msg);\n",
       "    // fig.ws.close()\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n",
       "    // Turn the data on the canvas into data in the output cell.\n",
       "    var width = this.canvas.width / this.ratio;\n",
       "    var dataURL = this.canvas.toDataURL();\n",
       "    this.cell_info[1]['text/html'] =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Tell IPython that the notebook contents must change.\n",
       "    IPython.notebook.set_dirty(true);\n",
       "    this.send_message('ack', {});\n",
       "    var fig = this;\n",
       "    // Wait a second, then push the new image to the DOM so\n",
       "    // that it is saved nicely (might be nice to debounce this).\n",
       "    setTimeout(function () {\n",
       "        fig.push_to_output();\n",
       "    }, 1000);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'btn-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'btn-group';\n",
       "    var button;\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'btn-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        button = fig.buttons[name] = document.createElement('button');\n",
       "        button.classList = 'btn btn-default';\n",
       "        button.href = '#';\n",
       "        button.title = name;\n",
       "        button.innerHTML = '<i class=\"fa ' + image + ' fa-lg\"></i>';\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    // Add the status bar.\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message pull-right';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "\n",
       "    // Add the close button to the window.\n",
       "    var buttongrp = document.createElement('div');\n",
       "    buttongrp.classList = 'btn-group inline pull-right';\n",
       "    button = document.createElement('button');\n",
       "    button.classList = 'btn btn-mini btn-primary';\n",
       "    button.href = '#';\n",
       "    button.title = 'Stop Interaction';\n",
       "    button.innerHTML = '<i class=\"fa fa-power-off icon-remove icon-large\"></i>';\n",
       "    button.addEventListener('click', function (_evt) {\n",
       "        fig.handle_close(fig, {});\n",
       "    });\n",
       "    button.addEventListener(\n",
       "        'mouseover',\n",
       "        on_mouseover_closure('Stop Interaction')\n",
       "    );\n",
       "    buttongrp.appendChild(button);\n",
       "    var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n",
       "    titlebar.insertBefore(buttongrp, titlebar.firstChild);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._remove_fig_handler = function (event) {\n",
       "    var fig = event.data.fig;\n",
       "    fig.close_ws(fig, {});\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (el) {\n",
       "    el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (el) {\n",
       "    // this is important to make the div 'focusable\n",
       "    el.setAttribute('tabindex', 0);\n",
       "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
       "    // off when our div gets focus\n",
       "\n",
       "    // location in version 3\n",
       "    if (IPython.notebook.keyboard_manager) {\n",
       "        IPython.notebook.keyboard_manager.register_events(el);\n",
       "    } else {\n",
       "        // location in version 2\n",
       "        IPython.keyboard_manager.register_events(el);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (event, _name) {\n",
       "    var manager = IPython.notebook.keyboard_manager;\n",
       "    if (!manager) {\n",
       "        manager = IPython.keyboard_manager;\n",
       "    }\n",
       "\n",
       "    // Check for shift+enter\n",
       "    if (event.shiftKey && event.which === 13) {\n",
       "        this.canvas_div.blur();\n",
       "        // select the cell after this one\n",
       "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
       "        IPython.notebook.select(index + 1);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    fig.ondownload(fig, null);\n",
       "};\n",
       "\n",
       "mpl.find_output_cell = function (html_output) {\n",
       "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
       "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
       "    // IPython event is triggered only after the cells have been serialised, which for\n",
       "    // our purposes (turning an active figure into a static one), is too late.\n",
       "    var cells = IPython.notebook.get_cells();\n",
       "    var ncells = cells.length;\n",
       "    for (var i = 0; i < ncells; i++) {\n",
       "        var cell = cells[i];\n",
       "        if (cell.cell_type === 'code') {\n",
       "            for (var j = 0; j < cell.output_area.outputs.length; j++) {\n",
       "                var data = cell.output_area.outputs[j];\n",
       "                if (data.data) {\n",
       "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
       "                    data = data.data;\n",
       "                }\n",
       "                if (data['text/html'] === html_output) {\n",
       "                    return [cell, data, j];\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "};\n",
       "\n",
       "// Register the function which deals with the matplotlib target/channel.\n",
       "// The kernel may be null if the page has been refreshed.\n",
       "if (IPython.notebook.kernel !== null) {\n",
       "    IPython.notebook.kernel.comm_manager.register_target(\n",
       "        'matplotlib',\n",
       "        mpl.mpl_figure_comm\n",
       "    );\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"\" width=\"640\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x28a1be96250>]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plt.plot(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "obvious-quarterly",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
